From 169d1c599ef277599a8d00cca6110c22b41770d7 Mon Sep 17 00:00:00 2001 From: 0x22 <0x22@futureclient.net> Date: Wed, 20 Nov 2024 07:54:56 -0800 Subject: [PATCH] Optimize UserDumpParser::GetMemBlock (#17) Co-authored-by: 0vercl0k <1476421+0vercl0k@users.noreply.github.com> --- .github/workflows/udmp-parser.yml | 161 +++++++++--------------------- src/CMakeLists.txt | 2 +- src/lib/udmp-parser.h | 20 ++-- src/parser/parser.cc | 9 +- src/python/CMakeLists.txt | 8 ++ src/python/pyproject.toml | 16 ++- src/python/tests/test_parser.py | 2 +- 7 files changed, 85 insertions(+), 133 deletions(-) diff --git a/.github/workflows/udmp-parser.yml b/.github/workflows/udmp-parser.yml index c4bb2df..cf711a9 100644 --- a/.github/workflows/udmp-parser.yml +++ b/.github/workflows/udmp-parser.yml @@ -2,174 +2,109 @@ name: Builds on: [push, pull_request] - jobs: parser: strategy: fail-fast: false matrix: variant: - - {os: windows-latest, generator: msvc, arch: x64, config: RelWithDebInfo} - - {os: windows-latest, generator: ninja, arch: x64, config: RelWithDebInfo} - - {os: windows-latest, generator: msvc, arch: win32, config: RelWithDebInfo} - - {os: windows-latest, generator: msvc, arch: arm64, config: RelWithDebInfo} - - {os: ubuntu-latest, generator: gcc, arch: x64, config: RelWithDebInfo} - - {os: ubuntu-latest, generator: clang, arch: x64, config: RelWithDebInfo} - - {os: macos-latest, generator: clang, arch: x64, config: Release} + # Available runners: https://github.com/actions/runner-images + - {os: windows-2019, generator: msvc, arch: x64, config: RelWithDebInfo, } + - {os: windows-2019, generator: msvc, arch: win32, config: RelWithDebInfo, } + - {os: windows-2019, generator: msvc, arch: arm64, config: RelWithDebInfo, } + - {os: ubuntu-22.04, generator: gcc, arch: x64, config: RelWithDebInfo, } + - {os: ubuntu-22.04, generator: clang, arch: x64, config: RelWithDebInfo, } + - {os: ubuntu-24.04, generator: gcc, arch: x64, config: RelWithDebInfo, } + - {os: ubuntu-24.04, generator: clang, arch: x64, config: RelWithDebInfo, } + - {os: macos-13, generator: clang, arch: x64, config: Release, } + # - {os: macos-13-xlarge, generator: clang, arch: arm64, config: Release, } # Paying runner + - {os: macos-14, generator: clang, arch: arm64, config: Release, } + # - {os: macos-14-large, generator: clang, arch: x64, config: Release, } # Paying runner runs-on: ${{ matrix.variant.os }} name: parser / ${{ matrix.variant.os }} / ${{ matrix.variant.generator }} / ${{ matrix.variant.arch }} env: - NB_CPU: 1 CMAKE_FLAGS: "-DBUILD_PARSER:BOOL=ON -DBUILD_PYTHON_BINDING:BOOL=OFF" + CMAKE_ARCH: "" + steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Environment Setup (Windows) - if: matrix.variant.os == 'windows-latest' + if: matrix.variant.os == 'windows-2019' run: | - echo "NB_CPU=$env:NUMBER_OF_PROCESSORS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - echo "CMAKE_ARCH='-A ${{ matrix.variant.arch }}'" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append Import-Module .\.github\Invoke-VisualStudio.ps1 Invoke-VisualStudio2022${{ matrix.variant.arch }} + echo "CMAKE_ARCH='-A ${{ matrix.variant.arch }}'" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Environment Setup (Linux) - if: matrix.variant.os == 'ubuntu-latest' + if: matrix.variant.os == 'ubuntu-22.04' run: | - sudo apt-get -y update - echo "NB_CPU=$(grep -c ^processor /proc/cpuinfo)" >> $GITHUB_ENV + sudo apt update - name: Environment Setup (Linux/GCC) - if: matrix.variant.os == 'ubuntu-latest' && matrix.variant.generator == 'gcc' + if: matrix.variant.os == 'ubuntu-22.04' && matrix.variant.generator == 'gcc' run: | - sudo apt install -y g++ ninja-build + sudo apt install -y g++ echo CC=gcc >> $GITHUB_ENV echo CXX=g++ >> $GITHUB_ENV - name: Environment Setup (Linux/CLang) - if: matrix.variant.os == 'ubuntu-latest' && matrix.variant.generator == 'clang' + if: matrix.variant.os == 'ubuntu-22.04' && matrix.variant.generator == 'clang' run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" echo CC=clang >> $GITHUB_ENV echo CXX=clang++ >> $GITHUB_ENV - - name: Environment Setup (MacOS) - if: matrix.variant.os == 'macos-latest' - run: | - echo NB_CPU=$(sysctl -n hw.ncpu) >> $GITHUB_ENV - - name: Build run: | mkdir build mkdir artifact cmake -S ./src -B ./build ${{ env.CMAKE_ARCH }} ${{ env.CMAKE_FLAGS }} - cmake --build ./build --verbose --config ${{ matrix.variant.config }} --parallel ${{ env.NB_CPU }} - cmake --install ./build --config ${{ matrix.variant.config }} --prefix ./artifact --verbose + cmake --build ./build --verbose --config ${{ matrix.variant.config }} + cmake --install ./build --config ${{ matrix.variant.config }} --prefix ./artifact - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: parser-${{ matrix.variant.os }}.${{ matrix.variant.generator }}-${{ matrix.variant.arch }}.${{ matrix.variant.config }} + name: parser-${{ matrix.variant.os }}.${{ matrix.variant.generator }}-${{ matrix.variant.arch }}.${{ matrix.variant.config }}-${{ github.sha }} path: artifact/ bindings: strategy: fail-fast: false matrix: - # nanobind does not support Python < 3.8. - python-version: ['3.8', '3.9', '3.10', '3.11'] variant: - - {os: windows-latest, generator: msvc, arch: x64, config: RelWithDebInfo, py-arch: x64} - - {os: windows-latest, generator: msvc, arch: win32, config: RelWithDebInfo, py-arch: x86} - # - {os: windows-latest, generator: msvc, arch: arm64, config: RelWithDebInfo, py-arch: x64} # Unsupported (see https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json) - - {os: ubuntu-latest, generator: gcc, arch: x64, config: RelWithDebInfo, py-arch: x64} - - {os: ubuntu-latest, generator: clang, arch: x64, config: RelWithDebInfo, py-arch: x64} - - {os: macos-latest, generator: clang, arch: x64, config: Release, py-arch: x64} + # Available runners: https://github.com/actions/runner-images + - {os: windows-2019, config: RelWithDebInfo } + - {os: ubuntu-22.04, config: RelWithDebInfo } + # most up to date free intel based osx? + - {os: macos-13, config: Release } runs-on: ${{ matrix.variant.os }} - name: bindings / ${{ matrix.variant.os }} / ${{ matrix.variant.generator }} / ${{ matrix.python-version }} / ${{ matrix.variant.arch }} + name: bindings / ${{ matrix.variant.os }} env: - NB_CPU: 1 CMAKE_FLAGS: "-DBUILD_PARSER:BOOL=OFF -DBUILD_PYTHON_BINDING:BOOL=ON" steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v4 + - name: Build / test wheels + uses: pypa/cibuildwheel@v2.21.3 with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.variant.py-arch }} - - - name: Install Python pre-requisites - run: | - python -m pip install --upgrade pip setuptools wheel - python -m pip install --user --upgrade -r src/python/requirements.txt - - - name: Environment Setup (Windows) - if: matrix.variant.os == 'windows-latest' - run: | - echo "NB_CPU=$env:NUMBER_OF_PROCESSORS" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - echo "CMAKE_ARCH='-A ${{ matrix.variant.arch }}'" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - Import-Module .\.github\Invoke-VisualStudio.ps1 - Invoke-VisualStudio2022${{ matrix.variant.arch }} - - - name: Environment Setup (Linux) - if: matrix.variant.os == 'ubuntu-latest' - run: | - sudo apt-get -y update - echo "NB_CPU=$(grep -c ^processor /proc/cpuinfo)" >> $GITHUB_ENV - - - name: Environment Setup (Linux/GCC) - if: matrix.variant.os == 'ubuntu-latest' && matrix.variant.generator == 'gcc' - run: | - sudo apt install -y g++ ninja-build - echo CC=gcc >> $GITHUB_ENV - echo CXX=g++ >> $GITHUB_ENV - - - name: Environment Setup (Linux/CLang) - if: matrix.variant.os == 'ubuntu-latest' && matrix.variant.generator == 'clang' - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" - echo CC=clang >> $GITHUB_ENV - echo CXX=clang++ >> $GITHUB_ENV - - - name: Environment Setup (MacOS) - if: matrix.variant.os == 'macos-latest' - run: | - echo NB_CPU=$(sysctl -n hw.ncpu) >> $GITHUB_ENV - - - name: Build - run: | - mkdir build - mkdir artifact - cmake -S ./src -B ./build ${{ env.CMAKE_ARCH }} ${{ env.CMAKE_FLAGS }} - cmake --build ./build --verbose --config ${{ matrix.variant.config }} --parallel ${{ env.NB_CPU }} - cmake --install ./build --config ${{ matrix.variant.config }} --prefix ./artifact --verbose - - - name: Python Binding Tests - run: | - cd src/python - python -m pip install -r tests/requirements.txt -U - python -m pip install --user -U . - pytest -vvv ./tests - cd ../.. - - - name: Build wheel - run: | - cd src/python - mkdir ../../wheel - python -m pip wheel . -w ../../wheel - cd ../.. + package-dir: ./src/python - name: Upload wheel - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels - path: wheel/*.whl - - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: python-${{ matrix.variant.os }}.${{ matrix.variant.generator }}-${{ matrix.variant.arch }}.${{ matrix.variant.config }} - path: artifact/ + name: wheels-${{ matrix.variant.os }} + path: ./wheelhouse/*.whl + merge: + runs-on: ubuntu-latest + needs: bindings + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: wheels + pattern: wheels-* diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cbf3f80..d37f44a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,7 +9,7 @@ project( udmp-parser DESCRIPTION "A Cross-Platform C++ parser library for Windows user minidumps." HOMEPAGE_URL https://github.com/0vercl0k/udmp-parser - VERSION 0.5.0 + VERSION 0.6.0 ) set(PROJECT_AUTHOR 0vercl0k) diff --git a/src/lib/udmp-parser.h b/src/lib/udmp-parser.h index 6814911..77b087d 100644 --- a/src/lib/udmp-parser.h +++ b/src/lib/udmp-parser.h @@ -90,7 +90,7 @@ static void DbgPrintf(const char *Format, ...) { struct Version { static inline const uint16_t Major = 0; - static inline const uint16_t Minor = 4; + static inline const uint16_t Minor = 6; static inline const std::string Release = ""; }; @@ -695,7 +695,7 @@ struct MemBlock_t { : BaseAddress(Info_.BaseAddress), AllocationBase(Info_.AllocationBase), AllocationProtect(Info_.AllocationProtect), RegionSize(Info_.RegionSize), State(Info_.State), - Protect(Info_.Protect), Type(Info_.Type){}; + Protect(Info_.Protect), Type(Info_.Type) {}; std::string to_string() const { std::stringstream ss; @@ -1000,14 +1000,16 @@ class UserDumpParser { } const MemBlock_t *GetMemBlock(const uint64_t Address) const { - const auto &Res = - std::find_if(Mem_.begin(), Mem_.end(), [&](const auto &It) { - return Address >= It.first && - Address < (It.first + It.second.RegionSize); - }); + auto It = Mem_.upper_bound(Address); + if (It == Mem_.begin()) { + return nullptr; + } - if (Res != Mem_.end()) { - return &Res->second; + It--; + const auto &[MemBlockAddress, MemBlock] = *It; + if (Address >= MemBlockAddress && + Address < (MemBlockAddress + MemBlock.RegionSize)) { + return &MemBlock; } return nullptr; diff --git a/src/parser/parser.cc b/src/parser/parser.cc index 25ce5f9..3685f8b 100644 --- a/src/parser/parser.cc +++ b/src/parser/parser.cc @@ -524,10 +524,11 @@ int main(int argc, char *argv[]) { // const auto BlockStart = Block->BaseAddress; - const auto BlockSize = Block->DataSize; - const auto BlockEnd = BlockStart + BlockSize; + const auto BlockDataSize = Block->DataSize; + const auto BlockRegionSize = Block->RegionSize; + const auto BlockEnd = BlockStart + BlockRegionSize; printf("%016" PRIx64 " -> %016" PRIx64 "\n", BlockStart, BlockEnd); - if (BlockSize > 0) { + if (BlockDataSize > 0) { // // Calculate where from we need to start dumping, and the appropriate @@ -535,7 +536,7 @@ int main(int argc, char *argv[]) { // const auto OffsetFromStart = DumpAddress - BlockStart; - const auto Remaining = size_t(BlockSize - OffsetFromStart); + const auto Remaining = size_t(BlockDataSize - OffsetFromStart); const size_t MaxSize = 0x100; const size_t DumpSize = std::min(MaxSize, Remaining); utils::Hexdump(BlockStart + OffsetFromStart, diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt index 4f2fabf..9e56461 100644 --- a/src/python/CMakeLists.txt +++ b/src/python/CMakeLists.txt @@ -6,6 +6,14 @@ # With contribution from: # * hugsy - (github.com/hugsy) # +cmake_minimum_required(VERSION 3.20) + +project( + udmp-parser-python + DESCRIPTION "A Cross-Platform C++ parser library for Windows user minidumps." + HOMEPAGE_URL https://github.com/0vercl0k/udmp-parser + VERSION 0.6.0 +) find_package(Python 3 REQUIRED COMPONENTS Interpreter Development.Module diff --git a/src/python/pyproject.toml b/src/python/pyproject.toml index 2f7a236..92a5a91 100644 --- a/src/python/pyproject.toml +++ b/src/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build" [project] name = "udmp-parser" -version = "0.5.0" +version = "0.6.0" description = "A Cross-Platform C++ parser library for Windows user minidumps." readme = "README.md" requires-python = ">=3.8" @@ -13,6 +13,11 @@ classifiers = [ "Development Status :: 4 - Beta", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Assemblers", "Natural Language :: English", ] @@ -28,7 +33,7 @@ generate_minidump = "udmp_parser.utils:generate_minidump_from_command_line" profile = "black" [tool.scikit-build] -wheel.py-api = "cp312" +wheel.py-api = "cp313" minimum-version = "0.4" build-dir = "build/{wheel_tag}" cmake.minimum-version = "3.20" @@ -36,8 +41,9 @@ cmake.args = ["-DBUILD_PYTHON_PACKAGE:BOOL=ON"] [tool.cibuildwheel] build-verbosity = 1 -test-command = "pytest {project}/tests" -test-requires = "pytest" +skip = "cp37-* pp* *musllinux*" +before-test = "pip install -U -r {project}/src/python/tests/requirements.txt" +test-command = "pytest -vvv {project}/src/python/tests" [tool.cibuildwheel.macos.environment] -MACOSX_DEPLOYMENT_TARGET = "10.14" +MACOSX_DEPLOYMENT_TARGET = "10.15" diff --git a/src/python/tests/test_parser.py b/src/python/tests/test_parser.py index ca6a47c..28fc70c 100644 --- a/src/python/tests/test_parser.py +++ b/src/python/tests/test_parser.py @@ -55,7 +55,7 @@ def tearDown(self) -> None: def test_version(self): assert udmp_parser.version.major == 0 - assert udmp_parser.version.minor == 4 + assert udmp_parser.version.minor == 6 assert udmp_parser.version.release == "" def test_parser_basic(self):