vm: fix llvm as a standalone library (#154)

* fix compile as a standalone library

* add ci for test

* fix ci

* Update cli compile doc

* fix libbpf

* fix vm cli

.github/workflows/test-llvm-jit.yml

@@ -24,7 +24,18 @@ jobs:
 
     - name: build runtime
       run:  CC=gcc-12 CXX=g++-12 make build-llvm -j
+
     - name: Run tests
       run: |
         ./build/vm/llvm-jit/unit-test/llvm_jit_tests
 
+    - name: build llvm JIT/AOT as a standalone library
+      run: |
+        cd vm/llvm-jit &&\
+        CC=gcc-12 CXX=g++-12 cmake -B build -DCMAKE_BUILD_TYPE=Release &&\
+        CC=gcc-12 CXX=g++-12 cmake --build build --target all -j
+
+    - name: build vm as a standalone library
+      run: |
+        cd vm && CC=gcc-12 CXX=g++-12 make build-llvm -j
+    

CMakeLists.txt

@@ -88,6 +88,9 @@ set(SPDLOG_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/spdlog/include)
 
 # main library
 add_subdirectory(vm)
+if (BPFTIME_LLVM_JIT)
+  add_subdirectory(vm/cli)
+endif()
 add_subdirectory(runtime)
 add_subdirectory(daemon)
 add_subdirectory(tools)

README.md

@@ -15,11 +15,11 @@
 
 ## Key Features
 
-- **Uprobe and Syscall hooks based on binary rewriting**: Run eBPF programs in userspace, attaching them to Uprobes and Syscall tracepoints: **No manual instrumentation or restart required!**. It can `trace`, `replace` or `patch` the execution of a function, `hook`, `filter` or `redirect` all syscalls of a process safely, and efficiently with an eBPF userspace runtime.
+- **Uprobe and Syscall hooks based on binary rewriting**: Run eBPF programs in userspace, attaching them to Uprobes and Syscall tracepoints: **No manual instrumentation or restart required!**. It can `trace` or `change` the execution of a function, `hook` or `filter` all syscalls of a process safely, and efficiently with an eBPF userspace runtime.
 - **Performance**: Experience up to a 10x speedup in Uprobe overhead compared to kernel uprobe and uretprobe.
 - **Interprocess eBPF Maps**: Implement userspace eBPF maps in shared userspace memory for summary aggregation or control plane communication.
 - **Compatibility**: use existing eBPF toolchains like clang and libbpf to develop userspace eBPF without any modifications. Supporting CO-RE via BTF, and offering userspace host function access.
-- **JIT Support**: Benefit from a cross-platform eBPF interpreter and a high-speed JIT compiler powered by LLVM. It also includes a handcrafted x86 JIT in C for limited resources.
+- **JIT Support**: Benefit from a cross-platform eBPF interpreter and a high-speed `JIT/AOT` compiler powered by LLVM. It also includes a handcrafted x86 JIT in C for limited resources. The vm can be built as `a standalone library` like ubpf.
 - **No instrumentation**: Can inject eBPF runtime into any running process without the need for a restart or manual recompilation.
 - **Run with kernel eBPF**: Can load userspace eBPF from kernel, and using kernel eBPF maps to cooperate with kernel eBPF programs like kprobes and network filters.
 
@@ -199,10 +199,7 @@ See [documents/build-and-test.md](https://eunomia.dev/bpftime/documents/build-an
 
 `bpftime` is continuously evolving with more features in the pipeline:
 
-- [X] ring buffer output support.
-- [X] perf event output support.
-- [X] Figure out how to run transparently with kernel probe
-- [X] An AOT compiler for eBPF can be easily added based on the LLVM IR.
+- [X] An AOT compiler for eBPF based on the LLVM.
 - [ ] More examples and usecases:
   - [ ] Network on userspace eBPF
   - [X] Hotpatch userspace application

vm/CMakeLists.txt

@@ -38,7 +38,6 @@ message(STATUS "Started CMake for ${PROJECT_NAME} v${PROJECT_VERSION}...\n")
 
 if(BPFTIME_LLVM_JIT)
   add_subdirectory(llvm-jit)
-  add_subdirectory(cli)
 else()
   add_subdirectory(simple-jit)
 endif()

vm/Makefile

@@ -58,6 +58,15 @@ build: ## build the package
 	cmake -Bbuild  -DBPFTIME_ENABLE_UNIT_TESTING=1
 	cmake --build build --config Debug
 
+build-ubpf: ## build the package with ubpf backend
+	rm -rf build/
+	cmake -Bbuild
+	cmake --build build --config Release
+
+build-llvm: ## build with llvm as jit backend
+	cmake -Bbuild -DBPFTIME_LLVM_JIT=1
+	cmake --build build --config Release
+
 build-arm64: ## build the package on arm64
 	rm -rf build/
 	cmake -Bbuild -DCMAKE_TOOLCHAIN_FILE=../cmake/aarch64-toolchain.cmake -DARCH=aarch64 -DBPFTIME_ENABLE_UNIT_TESTING=1

vm/README.md

@@ -1,27 +1,39 @@
-# bpftime vm: userspace eBPF vm with JIT support
+# bpftime vm: userspace eBPF vm with JIT/AOT support
 
-The bpf vm and jit for eBPF usersapce runtime.
+The bpf vm and JIT/AOT for eBPF usersapce runtime.
 
 you can choose from llvm-jit and a simple-jit/interpreter based on ubpf.
+The JIT can be built as a standalone library and integrated into other projects.
+You can also try the cli tool to compile and run AOT eBPF programs.
 
-## LLVM jit for eBPF
-
-see [llvm-jit/README.md](llvm-jit/README.md)
-
-## a simple jit modified from ubpf
+Build the vm only:
 
-see [simple-jit/README.md](simple-jit/README.md)
+```sh
+make build-llvm # build llvm backend
+make build-ubpf # build ubpf backend
+```
 
-## build
+## LLVM JIT/AOT for eBPF
 
-The JIT can be built as a standalone library and integrated into other projects.
+see [llvm-jit/README.md](llvm-jit/README.md).
 
-In `vm` directory, run:
+You can also build the llvm JIT/AOT for eBPF as a standalone library in it's own directory:
 
 ```sh
-make build
+sudo apt install llvm-15-dev
+cd llvm-jit
+cmake -B build -DCMAKE_BUILD_TYPE=Release
+cmake --build build --target all -j
 ```
 
+You can see the cli for how to use [AOT compile](cli/README.md).
+
+## a simple jit modified from ubpf
+
+see [simple-jit/README.md](simple-jit/README.md)
+
+> Note: we will use ubpf to replace the simple-jit in the future.
+
 ## Example Usage
 
 See [example/main.c](example/main.c) for how to use it.
@@ -35,10 +47,14 @@ $ bpftime-cli
 Usage: build/vm/cli/bpftime-cli <path to ebpf instructions> [path to memory for the ebpf program]
 ```
 
-## benchmark
+See [cli](cli/README.md) for more details. Since cli is dependent on libbpf for loading eBPF programs, you need to compile it from the project root:
 
-see [github.com/eunomia-bpf/bpf-benchmark](https://github.com/eunomia-bpf/bpf-benchmark) for how we evaluate and details.
+```sh
+make release-with-llvm-jit
+```
 
-## Roadmap
+See [.github/workflows/test-aot-cli.yml](../.github/workflows/test-aot-cli.yml) for more details.
 
-- [ ] AOT support for LLVM JIT
+## benchmark
+
+see [github.com/eunomia-bpf/bpf-benchmark](https://github.com/eunomia-bpf/bpf-benchmark) for how we evaluate and details.

vm/cli/main.cpp

@@ -16,6 +16,7 @@
 #include <unistd.h>
 #include <bpf/libbpf.h>
 #include <fstream>
+
 using namespace llvm;
 using namespace llvm::orc;
 
@@ -77,8 +78,10 @@ static int build_ebpf_program(const std::string &ebpf_elf,
 	}
 	return 0;
 }
+
 static ExitOnError exit_on_error;
 using bpf_func = uint64_t (*)(const void *, uint64_t);
+
 static int run_ebpf_program(const std::filesystem::path &elf,
 			    std::optional<std::string> memory)
 {
@@ -148,6 +151,7 @@ int main(int argc, const char **argv)
 		.nargs(0, 1);
 	program.add_subparser(build_command);
 	program.add_subparser(run_command);
+
 	try {
 		program.parse_args(argc, argv);
 	} catch (const std::exception &err) {

vm/llvm-jit/.gitignore

@@ -3,3 +3,4 @@ build
 /.vscode
 /.cache
 test*
+libvm-bpf.a

vm/llvm-jit/CMakeLists.txt

@@ -1,3 +1,5 @@
+cmake_minimum_required(VERSION 3.21)
+
 project(
     "llvm-bpf-jit"
     LANGUAGES C CXX
@@ -8,8 +10,6 @@ if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Debug")
 endif()
 
-cmake_minimum_required(VERSION 3.15)
-
 function(bpftime_setup_target target)
     set_property(TARGET ${target} PROPERTY CXX_STANDARD 20)
     target_include_directories(${target}
@@ -54,6 +54,13 @@ llvm_map_components_to_libnames(LLVM_LIBS
     nativecodegen
 )
 
+if(NOT DEFINED SPDLOG_INCLUDE)
+  message(INFO "Adding spdlo seperately..")
+  # spdlog
+  add_subdirectory(../../third_party/spdlog ${CMAKE_CURRENT_BINARY_DIR}/spdlog)
+  set(SPDLOG_INCLUDE ../../third_party/spdlog/include)
+endif()
+
 message(STATUS "LLVM_LIBS=${LLVM_LIBS}")
 
 target_link_libraries(vm-bpf ${LLVM_LIBS} spdlog::spdlog)
@@ -75,7 +82,10 @@ target_link_libraries(vm-llvm-bpf-test vm-bpf)
 
 target_include_directories(vm-bpf PUBLIC ../include include)
 
-add_subdirectory(unit-test)
+if(BPFTIME_ENABLE_UNIT_TESTING)
+  message(STATUS "Build unit tests for the project. Tests should always be found in the test folder\n")
+  add_subdirectory(unit-test)
+endif()
 
 include(FetchContent)
 FetchContent_Declare(

vm/llvm-jit/README.md

@@ -4,16 +4,16 @@ A faster and better support on different architectures jit based on LLVM.
 
 ## build
 
-```
-sudo apt install llvm-dev
+```sh
+sudo apt install llvm-15-dev
 cmake -B build -DCMAKE_BUILD_TYPE=Release
 cmake --build build --target all -j
 ```
 
 ## run
 
-```
-build/vm-llvm
+```sh
+build/vm-llvm-example
 ```
 
 ## Test with bpf-conformance
@@ -51,5 +51,5 @@ PASS: "tests/stxdw.data"
 PASS: "tests/stxh.data"
 PASS: "tests/stxw.data"
 PASS: "tests/subnet.data"
-Passed 147 out of 166 tests.
+Passed 165 out of 166 tests.
 ```

vm/llvm-jit/example/main-bpf-conformance.cpp

@@ -7,6 +7,7 @@
 #include <iostream>
 #include <vector>
 #include "ebpf-vm.h"
+
 /**
  * @brief Read in a string of hex bytes and return a vector of bytes.
  *

vm/llvm-jit/src/llvm/llvm_jit_context.cpp

@@ -37,6 +37,7 @@
 #include <picosha2.h>
 #include <boost/interprocess/sync/file_lock.hpp>
 #include <boost/interprocess/sync/scoped_lock.hpp>
+
 using namespace llvm;
 using namespace llvm::orc;
 using namespace bpftime;
@@ -144,6 +145,7 @@ ensure_aot_cache_dir_and_cache_file()
 
 	return { dir, cache_lock };
 }
+
 static std::optional<std::vector<uint8_t> >
 load_aot_cache(const std::filesystem::path &path)
 {
@@ -167,6 +169,7 @@ load_aot_cache(const std::filesystem::path &path)
 		return buffer;
 	}
 }
+
 ebpf_jit_fn llvm_bpf_jit_context::compile()
 {
 	spin_lock_guard guard(compiling.get());
@@ -386,6 +389,7 @@ llvm_bpf_jit_context::create_and_initialize_lljit_instance()
 	ExitOnErr(mainDylib.define(absoluteSymbols(lddwSyms)));
 	return { std::move(jit), extFuncNames, definedLddwHelpers };
 }
+
 ebpf_jit_fn llvm_bpf_jit_context::get_entry_address()
 {
 	if (!this->jit.has_value()) {

vm/llvm-jit/unit-test/llvm-aot.cpp

@@ -10,6 +10,7 @@ extern "C" uint64_t add_func(uint64_t a, uint64_t b, uint64_t, uint64_t,
 {
 	return a + b;
 }
+
 TEST_CASE("Test aot compilation")
 {
 	ebpf_vm *vm = ebpf_create();