Skip to content

Commit

Permalink
Merge pull request #17 from yomaytk/mnist
Browse files Browse the repository at this point in the history
Success to compile mnist aarch64 binary to WASM.
  • Loading branch information
yomaytk authored Mar 7, 2024
2 parents 6e748d6 + 85f8484 commit e6279f4
Show file tree
Hide file tree
Showing 36 changed files with 362 additions and 651 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ remill_disass.egg-info/*
*.app
*.wasm
elflift
test

*.pyc

Expand Down
1 change: 0 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,4 @@ add_subdirectory(backend/remill)
add_subdirectory(lifter)

# tests
add_subdirectory(tests/aarch64)
add_subdirectory(tests/elfconv)
15 changes: 13 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ echo "deb http://apt.llvm.org/${DISTRO_NAME}/ llvm-toolchain-${DISTRO_NAME}-${LL
echo "deb-src http://apt.llvm.org/${DISTRO_NAME}/ llvm-toolchain-${DISTRO_NAME}-${LLVM_VERSION} main" >> /etc/apt/sources.list

# several install
RUN apt update
RUN apt install -qqy --no-install-recommends file libtinfo-dev libzstd-dev python3-pip python3-setuptools python-setuptools python3 build-essential \
RUN apt-get update
RUN apt-get install -qqy --no-install-recommends file libtinfo-dev libzstd-dev python3-pip python3-setuptools python-setuptools python3 build-essential \
clang-${LLVM_VERSION} lld-${LLVM_VERSION} llvm-${LLVM_VERSION} ninja-build pixz xz-utils make rpm curl unzip tar git zip pkg-config vim \
libc6-dev liblzma-dev zlib1g-dev libselinux1-dev libbsd-dev ccache binutils-dev libelf-dev && \
apt upgrade --yes && apt clean --yes && \
Expand All @@ -46,6 +46,17 @@ RUN apt update && \
RUN cd /root && git clone https://github.com/emscripten-core/emsdk.git && cd emsdk && \
git pull && ./emsdk install latest && ./emsdk activate latest && . ./emsdk_env.sh && echo 'source "/root/emsdk/emsdk_env.sh"' >> /root/.bash_profile

# wasi-sdk install
# takes long times to build wasi-sdk in arm64 because wasi-sdk doesn't release arm64 packages.
RUN \
if [ "$( uname -m )" = "x86_64" ]; then \
cd /root && export WASI_VERSION=21 && export WASI_VERSION_FULL=${WASI_VERSION}.0 && echo -e "export WASI_VERSION=21\nexport WASI_VERSION_FULL=${WASI_VERSION}.0\nexport WASI_SDK_PATH=/root/wasi-sdk-${WASI_VERSION_FULL}" >> /root/.bash_profile && \
wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_VERSION}/wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz && tar xvf wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz && rm wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz; \
elif [ "$( uname -m )" = "aarch64" ]; then \
cd /root && git clone --recursive https://github.com/WebAssembly/wasi-sdk.git; \
cd wasi-sdk && NINJA_FLAGS=-v make package; \
fi

# WASI Runtimes install
RUN curl -sSf https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh | bash
RUN curl https://wasmtime.dev/install.sh -sSf | bash && echo 'export PATH=$PATH:/root/.wasmtime/bin' >> /root/.bash_profile
Expand Down
1 change: 1 addition & 0 deletions backend/remill/include/remill/BC/InstructionLifter.h
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ class InstructionLifter : public InstructionLifterIntf {
class Impl;

const std::unique_ptr<Impl> impl;
const std::string debug_memory_name;
const std::string debug_insn_name;
};

Expand Down
2 changes: 2 additions & 0 deletions backend/remill/include/remill/BC/TraceLifter.h
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ class TraceLifter::Impl {
max_inst_bytes(arch->MaxInstructionSize(arch->CreateInitialContext())),
indirectbr_block_name("L_indirectbr"),
g_get_jmp_block_address_func_name("__g_get_indirectbr_block_address"),
debug_memory_name("debug_memory"),
debug_insn_name("debug_insn"),
debug_call_stack_name("debug_call_stack") {
inst_bytes.reserve(max_inst_bytes);
Expand Down Expand Up @@ -234,6 +235,7 @@ class TraceLifter::Impl {

std::string indirectbr_block_name;
std::string g_get_jmp_block_address_func_name;
std::string debug_memory_name;
std::string debug_insn_name;
std::string debug_call_stack_name;
};
Expand Down
16 changes: 12 additions & 4 deletions backend/remill/lib/Arch/AArch64/Arch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -821,7 +821,7 @@ bool AArch64Arch::ArchDecodeInstruction(uint64_t address, std::string_view inst_
/* set operands of insn */
if (!aarch64::TryDecode(dinst, inst)) {
inst.category = Instruction::kCategoryInvalid;
printf("[WARNING] Unsupported instruction at address: 0x%08lx (TryDecode), instFrom: %s\n",
printf("[WARNING] Unsupported instruction at address: 0x%08lx (TryDecode), instForm: %s\n",
address, inst.function.c_str());
return false;
}
Expand Down Expand Up @@ -2940,9 +2940,9 @@ bool TryDecodeUCVTF_ASISDMISC_R(const InstData &data, Instruction &inst) {

// FRINTA <Dd>, <Dn>
bool TryDecodeFRINTA_D_FLOATDP1(const InstData &data, Instruction &inst) {
// AddRegOperand(inst, kActionWrite, kRegD, kUseAsValue, data.Rd);
// AddRegOperand(inst, kActionRead, kRegD, kUseAsValue, data.Rn);
return false;
AddRegOperand(inst, kActionWrite, kRegD, kUseAsValue, data.Rd);
AddRegOperand(inst, kActionRead, kRegD, kUseAsValue, data.Rn);
return true;
}

bool IsUnallocatedFloatEncoding(const InstData &data) {
Expand Down Expand Up @@ -3021,6 +3021,13 @@ bool TryDecodeFCVTZS_64D_FLOAT2INT(const InstData &data, Instruction &inst) {
return true;
}

// FCVTAS <Xd>, <Dn>
bool TryDecodeFCVTAS_64D_FLOAT2INT(const InstData &data, Instruction &inst) {
AddRegOperand(inst, kActionWrite, kRegX, kUseAsValue, data.Rd);
AddRegOperand(inst, kActionRead, kRegD, kUseAsValue, data.Rn);
return true;
}

// FCVTZU <Wd>, <Sn>
bool TryDecodeFCVTZU_32S_FLOAT2INT(const InstData &data, Instruction &inst) {
if (IsUnallocatedFloatEncoding(data)) {
Expand Down Expand Up @@ -3555,6 +3562,7 @@ static bool TryDecodeSTR_Vn_LDST_IMMPRE(const InstData &data, Instruction &inst,
AddPreIndexMemOp(inst, kActionWrite, num_bits, data.Rn, offset);
return true;
}

// STR <Qt>, [<Xn|SP>, #<simm>]!
bool TryDecodeSTR_Q_LDST_IMMPRE(const InstData &data, Instruction &inst) {
return TryDecodeSTR_Vn_LDST_IMMPRE(data, inst, kRegQ);
Expand Down
38 changes: 0 additions & 38 deletions backend/remill/lib/Arch/AArch64/Decode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29224,44 +29224,6 @@ bool TryDecodeFCVTAS_32D_FLOAT2INT(const InstData &, Instruction &) {
return false;
}

// FCVTAS FCVTAS_64D_float2int:
// 0 x Rd 0
// 1 x Rd 1
// 2 x Rd 2
// 3 x Rd 3
// 4 x Rd 4
// 5 x Rn 0
// 6 x Rn 1
// 7 x Rn 2
// 8 x Rn 3
// 9 x Rn 4
// 10 0
// 11 0
// 12 0
// 13 0
// 14 0
// 15 0
// 16 0 opcode 0
// 17 0 opcode 1
// 18 1 opcode 2
// 19 0 rmode 0
// 20 0 rmode 1
// 21 1
// 22 1 type 0
// 23 0 type 1
// 24 0
// 25 1
// 26 1
// 27 1
// 28 1
// 29 0 S 0
// 30 0
// 31 1 sf 0
// FCVTAS <Xd>, <Dn>
bool TryDecodeFCVTAS_64D_FLOAT2INT(const InstData &, Instruction &) {
return false;
}

// SSHLL SXTL_SSHLL_asimdshf_L:
// 0 x Rd 0
// 1 x Rd 1
Expand Down
24 changes: 24 additions & 0 deletions backend/remill/lib/Arch/AArch64/Semantics/CONVERT.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,15 @@ DEF_SEM(FCVTZS_Float64ToSInt64, R64W dst, V64 src) {
return memory;
}

// FCVTAS <Xd>, <Dn>
// (FIXME) not using rounding to nearest with ties to Away
DEF_SEM(FCVTAS_Float64ToSInt64, R64W dst, V64 src) {
auto float_val = FExtractV64(FReadV64(src), 0);
auto res = CheckedCast<float64_t, int64_t>(state, float_val);
WriteZExt(dst, res);
return memory;
}

DEF_SEM(FCVT_Float32ToFloat64, V128W dst, V32 src) {
auto float_val = FExtractV32(FReadV32(src), 0);
auto res = CheckedCast<float32_t, float64_t>(state, float_val);
Expand All @@ -128,6 +137,17 @@ DEF_SEM(FCVT_Float64ToFloat32, V128W dst, V64 src) {
FWriteV32(dst, res);
return memory;
}

// FRINTA <Dd>, <Dn>
// (FIXME) not using rounding to nearest with ties to Away
DEF_SEM(FRINTA_Float64ToSInt64, V64W dst, V64 src) {
auto float_val = FExtractV64(FReadV64(src), 0);
auto res = (double) (long) float_val;
// auto res = CheckedCast<float64_t, int64_t>(state, float_val);
FWriteV64(dst, res);
return memory;
}

} // namespace

// TODO(pag): UCVTF_H32_FLOAT2INT.
Expand All @@ -153,9 +173,13 @@ DEF_ISEL(FCVTZS_32S_FLOAT2INT) = FCVTZS_Float32ToSInt32;
DEF_ISEL(FCVTZS_32D_FLOAT2INT) = FCVTZS_Float64ToSInt32;
DEF_ISEL(FCVTZS_64D_FLOAT2INT) = FCVTZS_Float64ToSInt64;

DEF_ISEL(FCVTAS_64D_FLOAT2INT) = FCVTAS_Float64ToSInt64;

DEF_ISEL(FCVT_DS_FLOATDP1) = FCVT_Float32ToFloat64;
DEF_ISEL(FCVT_SD_FLOATDP1) = FCVT_Float64ToFloat32;

DEF_ISEL(FRINTA_D_FLOATDP1) = FRINTA_Float64ToSInt64;

namespace {

DEF_SEM(SCVTF_Int32ToFloat32, V128W dst, R32 src) {
Expand Down
7 changes: 7 additions & 0 deletions backend/remill/lib/Arch/AArch64/Semantics/DATAXFER.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,11 @@ DEF_SEM(StoreToOffset, S src, D base, ADDR offset) {
return memory;
}

DEF_SEM(StoreWordToOffset, V32 src, MV32W base, ADDR offset) {
FWriteV32(DisplaceAddress(base, Read(offset)), FReadV32(src));
return memory;
}

DEF_SEM(StoreDoubleToOffset, V64 src, MV64W base, ADDR offset) {
FWriteV64(DisplaceAddress(base, Read(offset)), FReadV64(src));
return memory;
Expand Down Expand Up @@ -212,6 +217,7 @@ DEF_ISEL(STRH_32_LDST_POS) = Store<R32, M16W>;

DEF_ISEL(STR_32_LDST_REGOFF) = StoreToOffset<R32, M32W>;
DEF_ISEL(STR_64_LDST_REGOFF) = StoreToOffset<R64, M64W>;
DEF_ISEL(STR_S_LDST_REGOFF) = StoreWordToOffset;
DEF_ISEL(STR_D_LDST_REGOFF) = StoreDoubleToOffset;

DEF_ISEL(SWP_32_MEMOP) = SWP_MEMOP<R32, R32W, M32, M32W>;
Expand Down Expand Up @@ -567,6 +573,7 @@ DEF_ISEL(FMOV_D64_FLOAT2INT) = FMOV_I64ToF64;

DEF_ISEL(FMOV_S_FLOATDP1) = FMOV_S;
DEF_ISEL(FMOV_D_FLOATDP1) = FMOV_D;

namespace {

DEF_SEM(ADRP, R64W dst, PC label) {
Expand Down
27 changes: 14 additions & 13 deletions backend/remill/lib/BC/InstructionLifter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ InstructionLifter::~InstructionLifter(void) {}

InstructionLifter::InstructionLifter(const Arch *arch_, const IntrinsicTable *intrinsics_)
: impl(new Impl(arch_, intrinsics_)),
debug_memory_name("debug_memory"),
debug_insn_name("debug_insn") {}

// Lift a single instruction into a basic block. `is_delayed` signifies that
Expand Down Expand Up @@ -102,8 +103,9 @@ LiftStatus InstructionLifter::LiftIntoBlock(Instruction &arch_inst, llvm::BasicB
isel_func = impl->unsupported_instruction;
arch_inst.operands.clear();
status = kLiftedUnsupportedInstruction;
printf("[WARNING] Unsupported instruction at address: 0x%08lx (SemanticsFunction)\n",
arch_inst.pc);
printf(
"[WARNING] Unsupported instruction at address: 0x%08lx (SemanticsFunction), instForm: %s\n",
arch_inst.pc, arch_inst.function.c_str());
}

llvm::IRBuilder<> ir(block);
Expand All @@ -122,17 +124,6 @@ LiftStatus InstructionLifter::LiftIntoBlock(Instruction &arch_inst, llvm::BasicB
ir.CreateStore(ir.CreateCall(impl->intrinsics->delay_slot_begin, temp_args), mem_ptr_ref);
}

/* append debug_insn function call */
if (UINT64_MAX != debug_insn_addr) {
llvm::IRBuilder<> __debug_ir(block);
auto _debug_insn_fn = module->getFunction(debug_insn_name);
if (!_debug_insn_fn) {
printf("[ERROR] debug_insn is undeclared.\n");
abort();
}
__debug_ir.CreateCall(_debug_insn_fn);
}

#if defined(LIFT_INSN_DEBUG)
do {
std::vector<uint64_t> target_addrs = {};
Expand Down Expand Up @@ -215,6 +206,16 @@ LiftStatus InstructionLifter::LiftIntoBlock(Instruction &arch_inst, llvm::BasicB
ir.CreateStore(ir.CreateCall(impl->intrinsics->delay_slot_end, temp_args), mem_ptr_ref);
}

/* append debug_insn function call */
if (UINT64_MAX != debug_insn_addr) {
llvm::IRBuilder<> __debug_ir(block);
auto _debug_insn_fn = module->getFunction(debug_insn_name);
auto _debug_memory_fn = module->getFunction(debug_memory_name);
CHECK(_debug_insn_fn && _debug_memory_fn);
__debug_ir.CreateCall(_debug_insn_fn);
__debug_ir.CreateCall(_debug_memory_fn);
}

return status;
}

Expand Down
43 changes: 27 additions & 16 deletions bin/elfconv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,15 @@ setting() {
ELFCONV_MACROS="-DELFCONV_BROWSER_ENV=1"
ELFCONV_DEBUG_MACROS=
ELFPATH=$( realpath "$1" )
WASMCC=$EMCC
WASMCCFLAGS=$EMCCFLAGS
WASMAR=$EMAR
WASISDKCXX=${WASI_SDK_PATH}/bin/clang++
WASISDKAR=${WASI_SDK_PATH}/bin/ar
WASISDKFLAGS="${OPTFLAGS} --sysroot=${WASI_SDK_PATH}/share/wasi-sysroot -I${ROOT_DIR}/backend/remill/include -I${ROOT_DIR}"

if [ "$TARGET" = "wasm-host" ]; then
ELFCONV_MACROS="-DELFC_RUNTIME_HOST_ENV=1"
ELFCONV_MACROS="-DELFC_WASI_ENV=1"
fi

}
Expand All @@ -33,22 +39,27 @@ main() {

# build runtime
echo "[INFO] Building elfconv-Runtime ..."
cd "${RUNTIME_DIR}"
$EMCC $EMCCFLAGS $ELFCONV_MACROS $ELFCONV_DEBUG_MACROS -o Entry.o -c Entry.cpp && \
$EMCC $EMCCFLAGS $ELFCONV_MACROS $ELFCONV_DEBUG_MACROS -o Memory.o -c Memory.cpp && \
$EMCC $EMCCFLAGS $ELFCONV_MACROS $ELFCONV_DEBUG_MACROS -o Syscall.o -c Syscall.cpp && \
$EMCC $EMCCFLAGS $ELFCONV_MACROS $ELFCONV_DEBUG_MACROS -o VmIntrinsics.o -c VmIntrinsics.cpp && \
$EMCC $EMCCFLAGS $ELFCONV_MACROS $ELFCONV_DEBUG_MACROS -o Util.o -c ${UTILS_DIR}/Util.cpp && \
$EMCC $EMCCFLAGS $ELFCONV_MACROS $ELFCONV_DEBUG_MACROS -o elfconv.o -c ${UTILS_DIR}/elfconv.cpp && \
$EMAR rcs libelfconv.a Entry.o Memory.o Syscall.o VmIntrinsics.o Util.o elfconv.o
if [ "$TARGET" = "wasm-host" ]; then
WASMCC=$WASISDKCXX
WASMCCFLAGS=$WASISDKFLAGS
WASMAR=$WASISDKAR
fi
cd "${RUNTIME_DIR}" || { echo "cd Failure"; exit 1; }
$WASMCC "$WASMCCFLAGS" $ELFCONV_MACROS "$ELFCONV_DEBUG_MACROS" -o Entry.o -c Entry.cpp && \
$WASMCC "$WASMCCFLAGS" $ELFCONV_MACROS "$ELFCONV_DEBUG_MACROS" -o Memory.o -c Memory.cpp && \
$WASMCC "$WASMCCFLAGS" $ELFCONV_MACROS "$ELFCONV_DEBUG_MACROS" -o Syscall.o -c Syscall.cpp && \
$WASMCC "$WASMCCFLAGS" $ELFCONV_MACROS "$ELFCONV_DEBUG_MACROS" -o VmIntrinsics.o -c VmIntrinsics.cpp && \
$WASMCC "$WASMCCFLAGS" $ELFCONV_MACROS "$ELFCONV_DEBUG_MACROS" -o Util.o -c "${UTILS_DIR}"/Util.cpp && \
$WASMCC "$WASMCCFLAGS" $ELFCONV_MACROS "$ELFCONV_DEBUG_MACROS" -o elfconv.o -c "${UTILS_DIR}"/elfconv.cpp && \
$WASMAR rcs libelfconv.a Entry.o Memory.o Syscall.o VmIntrinsics.o Util.o elfconv.o
mv libelfconv.a "${BIN_DIR}/"
rm *.o
echo "[INFO] Generate libelfconv.a."

# ELF -> LLVM bc
cp -p "${BUILD_LIFTER_DIR}/elflift" "${BIN_DIR}/"
echo "[INFO] Converting ELF to LLVM bitcode ..."
cd "${BIN_DIR}"
cd "${BIN_DIR}" || { echo "cd Failure"; exit 1; }
./elflift \
--arch aarch64 \
--bc_out lift.bc \
Expand All @@ -60,17 +71,17 @@ main() {
case "$TARGET" in
wasm-browser)
echo "[INFO] Converting LLVM bitcode to WASM binary (for browser) ..."
cd "${BIN_DIR}"
$EMCC -c lift.bc -o lift.o && \
$EMCC -o exe.wasm.html -L"./" -sWASM -sALLOW_MEMORY_GROWTH lift.o -lelfconv
cd "${BIN_DIR}" || { echo "cd Failure"; exit 1; }
$WASMCC -c lift.bc -o lift.o && \
$WASMCC -o exe.wasm.html -L"./" -sWASM -sALLOW_MEMORY_GROWTH lift.o -lelfconv
echo "[INFO] Generate WASM binary."
return 0
;;
wasm-host)
echo "[INFO] Converting LLVM bitcode to WASM binary (for server) ..."
cd "${BIN_DIR}"
$EMCC -c lift.bc -o lift.o && \
$EMCC -o exe.wasm -L"./" lift.o -lelfconv
cd "${BIN_DIR}" || { echo "cd Failure"; exit 1; }
$WASMCC -c lift.bc -o lift.o && \
$WASMCC -o exe.wasm -L"./" lift.o -lelfconv
echo "[INFO] Generate WASM binary."
return 0
;;
Expand Down
4 changes: 2 additions & 2 deletions lifter/Lift.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ int main(int argc, char *argv[]) {
if (!FLAGS_dbg_fun_cfg.empty()) {
for (auto &[fn_addr, dasm_func] : manager.disasm_funcs) {
/* append the address of necesarry debug function */
if (strncmp(dasm_func.func_name.substr(0, FLAGS_dbg_fun_cfg.length()).c_str(),
FLAGS_dbg_fun_cfg.c_str(), FLAGS_dbg_fun_cfg.length()) == 0) {
if (strncmp(dasm_func.func_name.substr(0, FLAGS_dbg_fun_cfg.length() + 4).c_str(),
(FLAGS_dbg_fun_cfg + "_____").c_str(), FLAGS_dbg_fun_cfg.length() + 4) == 0) {
control_flow_debug_list[fn_addr] = true;
break;
}
Expand Down
3 changes: 3 additions & 0 deletions lifter/MainLifter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,9 @@ llvm::Function *MainLifter::WrapImpl::DeclareDebugFunction() {
/* void debug_call_stack() */
llvm::Function::Create(llvm::FunctionType::get(llvm::Type::getVoidTy(context), {}, false),
llvm::Function::ExternalLinkage, debug_call_stack_name, *module);
// void debug_memory()
llvm::Function::Create(llvm::FunctionType::get(llvm::Type::getVoidTy(context), {}, false),
llvm::Function::ExternalLinkage, debug_memory_name, *module);
/* void debug_insn() */
return llvm::Function::Create(llvm::FunctionType::get(llvm::Type::getVoidTy(context), {}, false),
llvm::Function::ExternalLinkage, debug_insn_name, *module);
Expand Down
2 changes: 1 addition & 1 deletion lifter/TraceManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ bool AArch64TraceManager::isFunctionEntry(uint64_t addr) {

std::string AArch64TraceManager::GetUniqueLiftedFuncName(std::string func_name, uint64_t vma_s) {
std::stringstream lifted_fn_name;
lifted_fn_name << func_name << "_" << to_string(unique_i64++) << "_" << std::hex << vma_s;
lifted_fn_name << func_name << "_____" << to_string(unique_i64++) << "_" << std::hex << vma_s;
return lifted_fn_name.str();
}

Expand Down
Loading

0 comments on commit e6279f4

Please sign in to comment.