From e34a9e64686146967f9687f0287803df27f5694c Mon Sep 17 00:00:00 2001 From: momo5502 Date: Sun, 26 Jan 2025 07:04:41 +0100 Subject: [PATCH 1/4] Serialization fixes --- src/emulator/emulator.hpp | 5 ++++ src/emulator/memory_manager.cpp | 23 +++++++++++++------ src/emulator/memory_manager.hpp | 2 ++ src/emulator/serialization.hpp | 12 ++++++++++ .../serialization_test.cpp | 19 +++++++++++++++ src/windows-emulator/windows_emulator.cpp | 12 ++++++---- src/windows-emulator/windows_emulator.hpp | 4 +++- 7 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/emulator/emulator.hpp b/src/emulator/emulator.hpp index 858902e4..91fc0d80 100644 --- a/src/emulator/emulator.hpp +++ b/src/emulator/emulator.hpp @@ -55,6 +55,11 @@ class emulator : public cpu_interface, public memory_manager, public hook_interf void perform_deserialization(utils::buffer_deserializer& buffer, const bool is_snapshot) { + if (!is_snapshot) + { + this->unmap_all_memory(); + } + this->deserialize_state(buffer, is_snapshot); this->deserialize_memory_state(buffer, is_snapshot); } diff --git a/src/emulator/memory_manager.cpp b/src/emulator/memory_manager.cpp index 46c5f944..c16eddb7 100644 --- a/src/emulator/memory_manager.cpp +++ b/src/emulator/memory_manager.cpp @@ -94,6 +94,7 @@ namespace utils void memory_manager::serialize_memory_state(utils::buffer_serializer& buffer, const bool is_snapshot) const { + buffer.write_atomic(this->memory_layout_state_version_); buffer.write_map(this->reserved_regions_); if (is_snapshot) @@ -125,15 +126,10 @@ void memory_manager::deserialize_memory_state(utils::buffer_deserializer& buffer { if (!is_snapshot) { - for (const auto& reserved_region : this->reserved_regions_) - { - for (const auto& region : reserved_region.second.committed_regions) - { - this->unmap_memory(region.first, region.second.length); - } - } + assert(this->reserved_regions_.empty()); } + buffer.read_atomic(this->memory_layout_state_version_); buffer.read_map(this->reserved_regions_); if (is_snapshot) @@ -434,6 +430,19 @@ bool memory_manager::release_memory(const uint64_t address, size_t size) return true; } +void memory_manager::unmap_all_memory() +{ + for (const auto& reserved_region : this->reserved_regions_) + { + for (const auto& region : reserved_region.second.committed_regions) + { + this->unmap_memory(region.first, region.second.length); + } + } + + this->reserved_regions_.clear(); +} + uint64_t memory_manager::find_free_allocation_base(const size_t size, const uint64_t start) const { uint64_t start_address = std::max(MIN_ALLOCATION_ADDRESS, start ? start : 0x100000000ULL); diff --git a/src/emulator/memory_manager.hpp b/src/emulator/memory_manager.hpp index 39899a2c..9b6969c6 100644 --- a/src/emulator/memory_manager.hpp +++ b/src/emulator/memory_manager.hpp @@ -101,6 +101,8 @@ class memory_manager bool release_memory(uint64_t address, size_t size); + void unmap_all_memory(); + uint64_t allocate_memory(const size_t size, const memory_permission permissions, const bool reserve_only = false) { const auto allocation_base = this->find_free_allocation_base(size); diff --git a/src/emulator/serialization.hpp b/src/emulator/serialization.hpp index 10da2567..14e2b6af 100644 --- a/src/emulator/serialization.hpp +++ b/src/emulator/serialization.hpp @@ -157,6 +157,12 @@ namespace utils return object; } + template + void read_atomic(std::atomic& val) + { + val = this->read(); + } + template void read_optional(std::optional& val) { @@ -390,6 +396,12 @@ namespace utils } } + template + void write_atomic(const std::atomic& val) + { + this->write(val.load()); + } + template void write_optional(const std::optional& val) { diff --git a/src/windows-emulator-test/serialization_test.cpp b/src/windows-emulator-test/serialization_test.cpp index 73ca3e05..d86a4fde 100644 --- a/src/windows-emulator-test/serialization_test.cpp +++ b/src/windows-emulator-test/serialization_test.cpp @@ -2,6 +2,25 @@ namespace test { + TEST(SerializationTest, ResettingEmulatorWorks) + { + auto emu = create_sample_emulator(); + + utils::buffer_serializer serializer{}; + emu.serialize(serializer); + + emu.start(); + + ASSERT_TERMINATED_SUCCESSFULLY(emu); + + utils::buffer_deserializer deserializer{serializer.get_buffer()}; + emu.deserialize(deserializer); + + emu.start(); + + ASSERT_TERMINATED_SUCCESSFULLY(emu); + } + TEST(SerializationTest, SerializedDataIsReproducible) { auto emu1 = create_sample_emulator(); diff --git a/src/windows-emulator/windows_emulator.cpp b/src/windows-emulator/windows_emulator.cpp index e72a1795..242927e3 100644 --- a/src/windows-emulator/windows_emulator.cpp +++ b/src/windows-emulator/windows_emulator.cpp @@ -897,13 +897,13 @@ void windows_emulator::setup_process(const emulator_settings& settings) void windows_emulator::yield_thread() { - this->switch_thread = true; + this->switch_thread_ = true; this->emu().stop(); } void windows_emulator::perform_thread_switch() { - this->switch_thread = false; + this->switch_thread_ = false; while (!switch_to_next_thread(*this)) { // TODO: Optimize that @@ -931,7 +931,7 @@ void windows_emulator::on_instruction_execution(const uint64_t address) const auto thread_insts = ++thread.executed_instructions; if (thread_insts % MAX_INSTRUCTIONS_PER_TIME_SLICE == 0) { - this->switch_thread = true; + this->switch_thread_ = true; this->emu().stop(); } @@ -1088,14 +1088,14 @@ void windows_emulator::start(std::chrono::nanoseconds timeout, size_t count) while (true) { - if (this->switch_thread || !this->current_thread().is_thread_ready(*this)) + if (this->switch_thread_ || !this->current_thread().is_thread_ready(*this)) { this->perform_thread_switch(); } this->emu().start_from_ip(timeout, count); - if (!this->switch_thread && !this->emu().has_violation()) + if (!this->switch_thread_ && !this->emu().has_violation()) { break; } @@ -1128,6 +1128,7 @@ void windows_emulator::start(std::chrono::nanoseconds timeout, size_t count) void windows_emulator::serialize(utils::buffer_serializer& buffer) const { + buffer.write(this->switch_thread_); buffer.write(this->use_relative_time_); this->file_sys().serialize(buffer); this->emu().serialize(buffer); @@ -1145,6 +1146,7 @@ void windows_emulator::deserialize(utils::buffer_deserializer& buffer) return windows_emulator_wrapper{*this}; // }); + buffer.read(this->switch_thread_); buffer.read(this->use_relative_time_); this->file_sys().deserialize(buffer); diff --git a/src/windows-emulator/windows_emulator.hpp b/src/windows-emulator/windows_emulator.hpp index 332f6f4c..0c1c1a51 100644 --- a/src/windows-emulator/windows_emulator.hpp +++ b/src/windows-emulator/windows_emulator.hpp @@ -119,7 +119,6 @@ class windows_emulator bool verbose_calls{false}; bool buffer_stdout{false}; bool fuzzing{false}; - bool switch_thread{false}; void yield_thread(); void perform_thread_switch(); @@ -155,8 +154,11 @@ class windows_emulator file_system file_sys_; emulator_callbacks callbacks_{}; + + bool switch_thread_{false}; bool use_relative_time_{false}; bool silent_until_main_{false}; + std::unique_ptr emu_{}; std::vector syscall_hooks_{}; From 4597f3b6d6c7e101785add147730e63d78032ebe Mon Sep 17 00:00:00 2001 From: momo5502 Date: Sun, 26 Jan 2025 07:30:04 +0100 Subject: [PATCH 2/4] Prevent failure when accessing stack --- src/windows-emulator/syscall_dispatcher.cpp | 12 +++++++++--- src/windows-emulator/windows_emulator.cpp | 5 ++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/windows-emulator/syscall_dispatcher.cpp b/src/windows-emulator/syscall_dispatcher.cpp index 1ed7581c..810268ce 100644 --- a/src/windows-emulator/syscall_dispatcher.cpp +++ b/src/windows-emulator/syscall_dispatcher.cpp @@ -92,7 +92,8 @@ void syscall_dispatcher::dispatch(windows_emulator& win_emu) if (mod != context.ntdll && mod != context.win32u) { win_emu.callbacks().inline_syscall(syscall_id, address, mod ? mod->name.c_str() : "", - entry->second.name.c_str()); + entry->second.name); + win_emu.log.print(color::blue, "Executing inline syscall: %s (0x%X) at 0x%" PRIx64 " (%s)\n", entry->second.name.c_str(), syscall_id, address, mod ? mod->name.c_str() : ""); } @@ -101,7 +102,10 @@ void syscall_dispatcher::dispatch(windows_emulator& win_emu) if (mod->is_within(context.previous_ip)) { const auto rsp = c.emu.read_stack_pointer(); - const auto return_address = c.emu.read_memory(rsp); + + uint64_t return_address{}; + c.emu.try_read_memory(rsp, &return_address, sizeof(return_address)); + const auto* mod_name = context.mod_manager.find_name(return_address); win_emu.log.print(color::dark_gray, @@ -111,9 +115,11 @@ void syscall_dispatcher::dispatch(windows_emulator& win_emu) else { const auto* previous_mod = context.mod_manager.find_by_address(context.previous_ip); + win_emu.callbacks().outofline_syscall(syscall_id, address, mod ? mod->name.c_str() : "", - entry->second.name.c_str(), context.previous_ip, + entry->second.name, context.previous_ip, previous_mod ? previous_mod->name.c_str() : ""); + win_emu.log.print(color::blue, "Crafted out-of-line syscall: %s (0x%X) at 0x%" PRIx64 " (%s) via 0x%" PRIx64 " (%s)\n", diff --git a/src/windows-emulator/windows_emulator.cpp b/src/windows-emulator/windows_emulator.cpp index 242927e3..2f612d9f 100644 --- a/src/windows-emulator/windows_emulator.cpp +++ b/src/windows-emulator/windows_emulator.cpp @@ -960,7 +960,10 @@ void windows_emulator::on_instruction_execution(const uint64_t address) if (export_entry != binary->address_names.end()) { const auto rsp = this->emu().read_stack_pointer(); - const auto return_address = this->emu().read_memory(rsp); + + uint64_t return_address{}; + this->emu().try_read_memory(rsp, &return_address, sizeof(return_address)); + const auto* mod_name = this->process().mod_manager.find_name(return_address); log.print(is_interesting_call ? color::yellow : color::dark_gray, From 748574c7b4393dd5e1393cc2bcbd0247d0e09799 Mon Sep 17 00:00:00 2001 From: momo5502 Date: Sun, 26 Jan 2025 07:56:36 +0100 Subject: [PATCH 3/4] Fix emulator resetting --- src/windows-emulator-test/serialization_test.cpp | 6 +++--- src/windows-emulator/process_context.hpp | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/windows-emulator-test/serialization_test.cpp b/src/windows-emulator-test/serialization_test.cpp index d86a4fde..03f80e15 100644 --- a/src/windows-emulator-test/serialization_test.cpp +++ b/src/windows-emulator-test/serialization_test.cpp @@ -6,14 +6,14 @@ namespace test { auto emu = create_sample_emulator(); - utils::buffer_serializer serializer{}; - emu.serialize(serializer); + utils::buffer_serializer start_state{}; + emu.serialize(start_state); emu.start(); ASSERT_TERMINATED_SUCCESSFULLY(emu); - utils::buffer_deserializer deserializer{serializer.get_buffer()}; + utils::buffer_deserializer deserializer{start_state.get_buffer()}; emu.deserialize(deserializer); emu.start(); diff --git a/src/windows-emulator/process_context.hpp b/src/windows-emulator/process_context.hpp index 0a544474..a21ac6da 100644 --- a/src/windows-emulator/process_context.hpp +++ b/src/windows-emulator/process_context.hpp @@ -338,6 +338,11 @@ class moved_marker return this->was_moved_; } + void mark_as_moved() + { + this->was_moved_ = true; + } + private: bool was_moved_{false}; }; @@ -498,6 +503,11 @@ class emulator_thread : ref_counted_object buffer.read_vector(this->last_registers); } + void leak_memory() + { + this->marker.mark_as_moved(); + } + private: void setup_registers(x64_emulator& emu, const process_context& context) const; @@ -657,6 +667,11 @@ struct process_context buffer.read_vector(this->default_register_set); buffer.read(this->spawned_thread_count); + for (auto& thread : this->threads | std::views::values) + { + thread.leak_memory(); + } + buffer.read(this->threads); this->active_thread = this->threads.get(buffer.read()); From 82a99ee67af4b4b35351235732f2293460267c7c Mon Sep 17 00:00:00 2001 From: momo5502 Date: Sun, 26 Jan 2025 08:06:16 +0100 Subject: [PATCH 4/4] Assert identical state after reusing emulator instances --- src/windows-emulator-test/serialization_test.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/windows-emulator-test/serialization_test.cpp b/src/windows-emulator-test/serialization_test.cpp index 03f80e15..95461785 100644 --- a/src/windows-emulator-test/serialization_test.cpp +++ b/src/windows-emulator-test/serialization_test.cpp @@ -13,12 +13,20 @@ namespace test ASSERT_TERMINATED_SUCCESSFULLY(emu); + utils::buffer_serializer end_state1{}; + emu.serialize(end_state1); + utils::buffer_deserializer deserializer{start_state.get_buffer()}; emu.deserialize(deserializer); emu.start(); ASSERT_TERMINATED_SUCCESSFULLY(emu); + + utils::buffer_serializer end_state2{}; + emu.serialize(end_state2); + + ASSERT_EQ(end_state1.get_buffer(), end_state2.get_buffer()); } TEST(SerializationTest, SerializedDataIsReproducible)