From 2606ed3b4c5704a99d9ae9ec00f5d416e7c086fb Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:41:30 +0100 Subject: [PATCH 1/6] Provide PyOxidizer binaries --- .github/workflows/pyoxidizer.yml | 180 +++++++++++++++++++++++++++++++ acquire/acquire.py | 7 +- acquire/utils.py | 6 ++ pyoxidizer.bzl | 103 ++++++++++++++++++ 4 files changed, 290 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/pyoxidizer.yml create mode 100644 pyoxidizer.bzl diff --git a/.github/workflows/pyoxidizer.yml b/.github/workflows/pyoxidizer.yml new file mode 100644 index 00000000..3e33cc85 --- /dev/null +++ b/.github/workflows/pyoxidizer.yml @@ -0,0 +1,180 @@ +name: Build PyOxidizer Binary +on: + push: + tags: + - '*' + workflow_dispatch: + +jobs: + build-linux-gnu: + name: Build Linux (GNU) + runs-on: ubuntu-latest + container: quay.io/pypa/manylinux2014_x86_64 + + steps: + - uses: actions/checkout@v3 + + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pyoxidizer/ + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: linux-cache + + - name: Install dependencies + run: /opt/python/cp39-cp39/bin/pip install pyoxidizer dissect.target + + - name: Build binary + run: | + mkdir -p build/lib/dissect/target/plugins + /opt/python/cp39-cp39/bin/target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py + /opt/python/cp39-cp39/bin/pyoxidizer build --release --target-triple x86_64-unknown-linux-gnu --var flavor standalone + strip build/x86_64-unknown-linux-gnu/release/install/acquire + + - name: Verify binary + run: build/x86_64-unknown-linux-gnu/release/install/acquire --version + + - uses: actions/upload-artifact@v3 + with: + name: acquire-linux + path: build/x86_64-unknown-linux-gnu/release/install/* + + build-linux-musl: + name: Build Linux (musl) + runs-on: ubuntu-latest + container: quay.io/pypa/manylinux2014_x86_64 + env: + musl_version: 1.2.4 + + steps: + - uses: actions/checkout@v3 + + - name: Setup Rust + run: | + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable + echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV + + - uses: Swatinem/rust-cache@v2 + + - uses: actions/cache@v3 + with: + path: | + ~/.cache/pyoxidizer/ + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: linux-musl-cache + + - name: Setup musl + run: | + curl -L https://www.musl-libc.org/releases/musl-${{ env.musl_version }}.tar.gz | tar -xzf - + cd musl-* + ./configure --exec-prefix=/usr/local + make + make install + + - name: Install forked PyOxidizer + run: cargo install --git https://github.com/Schamper/pyoxidizer pyoxidizer --force --locked + + - name: Install dependencies + run: /opt/python/cp39-cp39/bin/pip install dissect.target + + - name: Build binary + run: | + mkdir -p build/lib/dissect/target/plugins + /opt/python/cp39-cp39/bin/target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py + pyoxidizer build --release --target-triple x86_64-unknown-linux-musl --var flavor standalone + strip build/x86_64-unknown-linux-musl/release/install/acquire + + - name: Verify binary + run: build/x86_64-unknown-linux-musl/release/install/acquire --version + + - uses: actions/upload-artifact@v3 + with: + name: acquire-linux-musl + path: build/x86_64-unknown-linux-musl/release/install/* + + build-windows: + name: Build Windows + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/cache@v3 + with: + path: | + ~\AppData\Local\pyoxidizer\ + ~\.cargo\bin\ + ~\.cargo\registry\index\ + ~\.cargo\registry\cache\ + ~\.cargo\git\db\ + key: windows-cache + + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: pip install pyoxidizer dissect.target + + - name: Build binary + run: | + mkdir -p build/lib/dissect/target/plugins + target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py + pyoxidizer build --release --target-triple x86_64-pc-windows-msvc --var flavor standalone_static + strip build/x86_64-pc-windows-msvc/release/install/acquire.exe + + - name: Verify binary + run: build/x86_64-pc-windows-msvc/release/install/acquire.exe --version + + - uses: actions/upload-artifact@v3 + with: + name: acquire-windows + path: build/x86_64-pc-windows-msvc/release/install/* + + build-macos: + name: Build macOS + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/cache@v3 + with: + path: | + ~/Library/Caches/pyoxidizer/ + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: macos-cache + + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: pip install pyoxidizer dissect.target + + - name: Build binary + run: | + mkdir -p build/lib/dissect/target/plugins + target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py + pyoxidizer build --release --target-triple x86_64-apple-darwin --var flavor standalone + pyoxidizer build --release --target-triple aarch64-apple-darwin --var flavor standalone + + mkdir -p build/universal2-apple-darwin/release/install + cp -r build/x86_64-apple-darwin/release/install/* build/universal2-apple-darwin/release/install/ + lipo -create build/x86_64-apple-darwin/release/install/acquire build/aarch64-apple-darwin/release/install/acquire -output build/universal2-apple-darwin/release/install/acquire + strip build/universal2-apple-darwin/release/install/acquire + + - name: Verify binary + run: build/universal2-apple-darwin/release/install/acquire --version + + - uses: actions/upload-artifact@v3 + with: + name: acquire-macos + path: build/universal2-apple-darwin/release/install/* diff --git a/acquire/acquire.py b/acquire/acquire.py index 6b281689..5bd9731c 100644 --- a/acquire/acquire.py +++ b/acquire/acquire.py @@ -42,6 +42,7 @@ from acquire.uploaders.plugin import UploaderPlugin, upload_files_using_uploader from acquire.uploaders.plugin_registry import UploaderRegistry from acquire.utils import ( + VERSION, check_and_set_acquire_args, check_and_set_log_args, create_argument_parser, @@ -56,11 +57,6 @@ persist_execution_report, ) -try: - from acquire.version import version -except ImportError: - version = "0.0.dev" - try: # Injected by pystandalone builder from acquire.config import CONFIG @@ -68,7 +64,6 @@ CONFIG = defaultdict(lambda: None) -VERSION = version ACQUIRE_BANNER = r""" _ __ _ ___ __ _ _ _(_)_ __ ___ diff --git a/acquire/utils.py b/acquire/utils.py index 673cc2c0..52f8ad5e 100644 --- a/acquire/utils.py +++ b/acquire/utils.py @@ -17,6 +17,11 @@ from acquire.outputs import OUTPUTS from acquire.uploaders.plugin_registry import UploaderRegistry +try: + from acquire.version import version as VERSION +except ImportError: + VERSION = "0.0.dev" + # Acquire Configuration for CAgent and TargetD CAGENT_TARGETD_ATTRS = { "cagent_key", @@ -148,6 +153,7 @@ def create_argument_parser(profiles: dict, modules: dict) -> argparse.ArgumentPa parser.add_argument(*args, **kwargs) parser.add_argument("-v", "--verbose", action="count", default=3, help="increase output verbosity") + parser.add_argument("--version", action="version", version=f"%(prog)s {VERSION}") return parser diff --git a/pyoxidizer.bzl b/pyoxidizer.bzl new file mode 100644 index 00000000..7999cc90 --- /dev/null +++ b/pyoxidizer.bzl @@ -0,0 +1,103 @@ +def make_exe(): + dist = default_python_distribution(flavor=VARS["flavor"]) + + policy = dist.make_python_packaging_policy() + policy.bytecode_optimize_level_two = True + policy.file_scanner_classify_files = True + policy.resources_location = "in-memory" + + python_config = dist.make_python_interpreter_config() + python_config.oxidized_importer = True + python_config.filesystem_importer = False + python_config.run_module = "acquire.acquire" + + exe = dist.to_python_executable( + name="acquire", + packaging_policy=policy, + config=python_config, + ) + exe.windows_runtime_dlls_mode = "when-present" + + # The default dependency list of acquire doesn't include enough, and full includes some that are hard to package + pip_args = [ + "acquire", + "dissect.cstruct", + "dissect.eventlog", + "dissect.evidence", + "dissect.extfs", + "dissect.fat", + "dissect.ffs", + "dissect.hypervisor", + "dissect.ntfs", + "dissect.regf", + "dissect.squashfs", + "dissect.target", + "dissect.util", + "dissect.vmfs", + "dissect.volume", + "dissect.xfs", + "minio", + ] + + # Lie about our platform to get cross-compilation to work (msgpack fails to download otherwise) + if BUILD_TARGET_TRIPLE == "x86_64-pc-windows-msvc": + pip_args += ["--platform", "win_amd64"] + elif BUILD_TARGET_TRIPLE == "i686-pc-windows-msvc": + pip_args += ["--platform", "win32"] + elif BUILD_TARGET_TRIPLE == "x86_64-unknown-linux-musl": + pip_args += ["--platform", "manylinux2014_x86_64"] + + # Use pip_download for all the dependencies + for resource in exe.pip_download(pip_args): + # Discard msgpack's extension, it has a pure Python fallback + if resource.name == "msgpack._cmsgpack": + continue + + # The crypto portions of minio aren't needed for normal usage + if resource.name == "_cffi_backend" or resource.name.startswith("_argon2_cffi_bindings"): + continue + + # Discard pycryptodome fully for the time being, unsure how to make it play nicely + if resource.name.startswith("Crypto"): + continue + + exe.add_python_resource(resource) + + # Add the _pluginlist.py "overlay" + # This is created by the CI, if you want to build manually, be sure to generate it: + # mkdir -p build/lib/dissect/target/plugins/ && target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py + exe.add_python_resources(exe.read_package_root("build/lib", ["dissect"])) + + # If you want to add your own configuration customizations, you can put them in here + # 'arguments' allows you to override specific arguments by default, e.g. ['--compress'] + # 'public_key' allows you to include a PEM encoded RSA public key for output encryption + # NOTE: pycryptodome is not currently packaged in this PyOxidizer configuration, so output encryption is unavailable + # 'upload' allows you to configure upload credentials + # Example AWS S3 configuration: {'mode': 'cloud', 'endpoint': 's3.amazonaws.com', 'access_id': '', 'access_key': '', 'bucket': ''} + exe.add_python_resource( + exe.make_python_module_source( + "acquire.config", + "CONFIG = {'arguments': [], 'public_key': '', 'upload': {}}", + False + ) + ) + + return exe + + +def make_embedded_resources(exe): + return exe.to_embedded_resources() + + +def make_install(exe): + files = FileManifest() + files.add_python_resource(".", exe) + + return files + + +register_target("exe", make_exe) +register_target("resources", make_embedded_resources, depends=["exe"], default_build_script=True) +register_target("install", make_install, depends=["exe"], default=True) + +resolve_targets() From 6adaece80a768aeff696f4a231e91f19862709d6 Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:58:03 +0100 Subject: [PATCH 2/6] Add comment about building acquire from source --- pyoxidizer.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyoxidizer.bzl b/pyoxidizer.bzl index 7999cc90..52553b28 100644 --- a/pyoxidizer.bzl +++ b/pyoxidizer.bzl @@ -39,6 +39,9 @@ def make_exe(): "minio", ] + # If you want to build acquire from the local source directory, uncomment this and remove "acquire" from pip_args + # exe.add_python_resources(exe.read_package_root(CWD, ["acquire"])) + # Lie about our platform to get cross-compilation to work (msgpack fails to download otherwise) if BUILD_TARGET_TRIPLE == "x86_64-pc-windows-msvc": pip_args += ["--platform", "win_amd64"] From 961fedbdb681854a4aa19eccaa94d79b0787efbc Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:17:59 +0100 Subject: [PATCH 3/6] Remove duplicate Rust cache, probably unnecessary? --- .github/workflows/pyoxidizer.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pyoxidizer.yml b/.github/workflows/pyoxidizer.yml index 3e33cc85..ac1be8e7 100644 --- a/.github/workflows/pyoxidizer.yml +++ b/.github/workflows/pyoxidizer.yml @@ -57,8 +57,6 @@ jobs: curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV - - uses: Swatinem/rust-cache@v2 - - uses: actions/cache@v3 with: path: | From dc4923305026a30ac5b04cabab98d208d3444e49 Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:29:58 +0100 Subject: [PATCH 4/6] Update PyOxidizer dependencies and build process --- .github/workflows/pyoxidizer.yml | 33 ++++++++++++++++++++------------ pyoxidizer.bzl | 2 +- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pyoxidizer.yml b/.github/workflows/pyoxidizer.yml index ac1be8e7..125e2757 100644 --- a/.github/workflows/pyoxidizer.yml +++ b/.github/workflows/pyoxidizer.yml @@ -1,9 +1,10 @@ name: Build PyOxidizer Binary on: - push: - tags: - - '*' workflow_dispatch: + inputs: + tag: + description: Tag to build + required: true jobs: build-linux-gnu: @@ -12,6 +13,8 @@ jobs: container: quay.io/pypa/manylinux2014_x86_64 steps: + # We use older versions of the checkout and cache actions, those work with the LIBC version of this container + # We also can't use the setup-python action for this reason - it depends on a newer LIBC - uses: actions/checkout@v3 - uses: actions/cache@v3 @@ -24,14 +27,15 @@ jobs: ~/.cargo/git/db/ key: linux-cache + # We need target-query to build the plugin list, let acquire determine which version to pull in - name: Install dependencies - run: /opt/python/cp39-cp39/bin/pip install pyoxidizer dissect.target + run: /opt/python/cp39-cp39/bin/pip install pyoxidizer acquire==${{ github.event.inputs.tag }} - name: Build binary run: | mkdir -p build/lib/dissect/target/plugins /opt/python/cp39-cp39/bin/target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py - /opt/python/cp39-cp39/bin/pyoxidizer build --release --target-triple x86_64-unknown-linux-gnu --var flavor standalone + /opt/python/cp39-cp39/bin/pyoxidizer build --release --target-triple x86_64-unknown-linux-gnu --var flavor standalone --var version ${{ github.event.inputs.tag }} strip build/x86_64-unknown-linux-gnu/release/install/acquire - name: Verify binary @@ -50,6 +54,8 @@ jobs: musl_version: 1.2.4 steps: + # We use older versions of the checkout and cache actions, those work with the LIBC version of this container + # We also can't use the setup-python action for this reason - it depends on a newer LIBC - uses: actions/checkout@v3 - name: Setup Rust @@ -78,14 +84,15 @@ jobs: - name: Install forked PyOxidizer run: cargo install --git https://github.com/Schamper/pyoxidizer pyoxidizer --force --locked + # We need target-query to build the plugin list, let acquire determine which version to pull in - name: Install dependencies - run: /opt/python/cp39-cp39/bin/pip install dissect.target + run: /opt/python/cp39-cp39/bin/pip install acquire==${{ github.event.inputs.tag }} - name: Build binary run: | mkdir -p build/lib/dissect/target/plugins /opt/python/cp39-cp39/bin/target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py - pyoxidizer build --release --target-triple x86_64-unknown-linux-musl --var flavor standalone + pyoxidizer build --release --target-triple x86_64-unknown-linux-musl --var flavor standalone --var version ${{ github.event.inputs.tag }} strip build/x86_64-unknown-linux-musl/release/install/acquire - name: Verify binary @@ -116,14 +123,15 @@ jobs: with: python-version: '3.9' + # We need target-query to build the plugin list, let acquire determine which version to pull in - name: Install dependencies - run: pip install pyoxidizer dissect.target + run: pip install pyoxidizer acquire==${{ github.event.inputs.tag }} - name: Build binary run: | mkdir -p build/lib/dissect/target/plugins target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py - pyoxidizer build --release --target-triple x86_64-pc-windows-msvc --var flavor standalone_static + pyoxidizer build --release --target-triple x86_64-pc-windows-msvc --var flavor standalone_static --var version ${{ github.event.inputs.tag }} strip build/x86_64-pc-windows-msvc/release/install/acquire.exe - name: Verify binary @@ -154,15 +162,16 @@ jobs: with: python-version: '3.9' + # We need target-query to build the plugin list, let acquire determine which version to pull in - name: Install dependencies - run: pip install pyoxidizer dissect.target + run: pip install pyoxidizer acquire==${{ github.event.inputs.tag }} - name: Build binary run: | mkdir -p build/lib/dissect/target/plugins target-build-pluginlist > build/lib/dissect/target/plugins/_pluginlist.py - pyoxidizer build --release --target-triple x86_64-apple-darwin --var flavor standalone - pyoxidizer build --release --target-triple aarch64-apple-darwin --var flavor standalone + pyoxidizer build --release --target-triple x86_64-apple-darwin --var flavor standalone --var version ${{ github.event.inputs.tag }} + pyoxidizer build --release --target-triple aarch64-apple-darwin --var flavor standalone --var version ${{ github.event.inputs.tag }} mkdir -p build/universal2-apple-darwin/release/install cp -r build/x86_64-apple-darwin/release/install/* build/universal2-apple-darwin/release/install/ diff --git a/pyoxidizer.bzl b/pyoxidizer.bzl index 52553b28..f9999e35 100644 --- a/pyoxidizer.bzl +++ b/pyoxidizer.bzl @@ -20,7 +20,7 @@ def make_exe(): # The default dependency list of acquire doesn't include enough, and full includes some that are hard to package pip_args = [ - "acquire", + "acquire==" + VARS["version"], "dissect.cstruct", "dissect.eventlog", "dissect.evidence", From 527e0e5d1cfd0ff72a32862079d043db6fd54781 Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:34:10 +0100 Subject: [PATCH 5/6] Install dissect package instead of acquire --- .github/workflows/pyoxidizer.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pyoxidizer.yml b/.github/workflows/pyoxidizer.yml index 125e2757..b9806bb4 100644 --- a/.github/workflows/pyoxidizer.yml +++ b/.github/workflows/pyoxidizer.yml @@ -27,9 +27,8 @@ jobs: ~/.cargo/git/db/ key: linux-cache - # We need target-query to build the plugin list, let acquire determine which version to pull in - name: Install dependencies - run: /opt/python/cp39-cp39/bin/pip install pyoxidizer acquire==${{ github.event.inputs.tag }} + run: /opt/python/cp39-cp39/bin/pip install pyoxidizer dissect - name: Build binary run: | @@ -84,9 +83,8 @@ jobs: - name: Install forked PyOxidizer run: cargo install --git https://github.com/Schamper/pyoxidizer pyoxidizer --force --locked - # We need target-query to build the plugin list, let acquire determine which version to pull in - name: Install dependencies - run: /opt/python/cp39-cp39/bin/pip install acquire==${{ github.event.inputs.tag }} + run: /opt/python/cp39-cp39/bin/pip install dissect - name: Build binary run: | @@ -123,9 +121,8 @@ jobs: with: python-version: '3.9' - # We need target-query to build the plugin list, let acquire determine which version to pull in - name: Install dependencies - run: pip install pyoxidizer acquire==${{ github.event.inputs.tag }} + run: pip install pyoxidizer dissect - name: Build binary run: | @@ -162,9 +159,8 @@ jobs: with: python-version: '3.9' - # We need target-query to build the plugin list, let acquire determine which version to pull in - name: Install dependencies - run: pip install pyoxidizer acquire==${{ github.event.inputs.tag }} + run: pip install pyoxidizer dissect - name: Build binary run: | From 122daee1649ab0ee62bd3ac5b52132b1071cfe4b Mon Sep 17 00:00:00 2001 From: Erik Schamper <1254028+Schamper@users.noreply.github.com> Date: Fri, 26 Jul 2024 11:56:25 +0000 Subject: [PATCH 6/6] Update PyOxidizer fork --- .github/workflows/pyoxidizer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyoxidizer.yml b/.github/workflows/pyoxidizer.yml index b9806bb4..e465b1ba 100644 --- a/.github/workflows/pyoxidizer.yml +++ b/.github/workflows/pyoxidizer.yml @@ -81,7 +81,7 @@ jobs: make install - name: Install forked PyOxidizer - run: cargo install --git https://github.com/Schamper/pyoxidizer pyoxidizer --force --locked + run: cargo install --git https://github.com/fox-it/pyoxidizer --branch esxi-compatibility pyoxidizer --force --locked - name: Install dependencies run: /opt/python/cp39-cp39/bin/pip install dissect