From 827aacab2bf09a27046801992f78f42447b17150 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:51:06 +0000 Subject: [PATCH 1/3] submod --- .gitmodules | 3 +++ firedancer | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 firedancer diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..7a37d8a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "firedancer"] + path = firedancer + url = git@github.com:firedancer-io/firedancer.git diff --git a/firedancer b/firedancer new file mode 160000 index 0000000..266734b --- /dev/null +++ b/firedancer @@ -0,0 +1 @@ +Subproject commit 266734b3354080724b3cb8140977743b692318bd From c7fc329adfe601481921148275638b4086aff07c Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 4 Jan 2024 20:14:04 +0000 Subject: [PATCH 2/3] maybe working already --- .github/actions/deps/action.yml | 54 ++++++++++ .github/actions/hugepages/action.yml | 18 ++++ .github/workflows/build_and_push.yml | 42 ++++++++ .gitignore | 2 + firedancer | 2 +- rust/.gitignore | 1 + rust/diff-sbpf_loader/Cargo.toml | 16 +++ rust/diff-sbpf_loader/build.rs | 3 + rust/diff-sbpf_loader/src/lib.rs | 148 +++++++++++++++++++++++++++ 9 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 .github/actions/deps/action.yml create mode 100644 .github/actions/hugepages/action.yml create mode 100644 .github/workflows/build_and_push.yml create mode 100644 .gitignore create mode 100644 rust/.gitignore create mode 100644 rust/diff-sbpf_loader/Cargo.toml create mode 100644 rust/diff-sbpf_loader/build.rs create mode 100644 rust/diff-sbpf_loader/src/lib.rs diff --git a/.github/actions/deps/action.yml b/.github/actions/deps/action.yml new file mode 100644 index 0000000..931ce66 --- /dev/null +++ b/.github/actions/deps/action.yml @@ -0,0 +1,54 @@ +name: deps +description: 'Build and cache dependencies' +inputs: + deps-script-path: + description: 'Path of deps.sh script' + required: true + default: './firedancer/deps.sh' + deps-bundle-path: + description: 'Path of deps-bundle.sh script' + required: true + default: './firedancer/contrib/deps-bundle.sh' +outputs: {} +runs: + using: composite + steps: + - name: Has apt-get? + shell: bash + run: | + if command -v apt-get > /dev/null 2>&1; then + echo "HAS_APT_GET=1" >> $GITHUB_ENV + else + echo "HAS_APT_GET=0" >> $GITHUB_ENV + fi + + - name: apt-get update + shell: bash + run: sudo apt-get update + if: env.HAS_APT_GET == '1' + + - id: deps-sh-hash + shell: bash + run: sha256sum '${{ inputs.deps-script-path }}' | awk '{print "HASH=" $1}' >> "$GITHUB_OUTPUT" + + - id: deps-sh-cache + uses: actions/cache@v3 + with: + path: deps-bundle.tar.zst + key: ${{ runner.os }}-deps-sh-${{ steps.deps-sh-hash.outputs.HASH }} + + - name: Install system level dependencies + shell: bash + run: FD_AUTO_INSTALL_PACKAGES=1 '${{ inputs.deps-script-path }}' check + + - name: Install dependencies from cache + shell: bash + run: tar -Izstd -xvf deps-bundle.tar.zst + if: steps.deps-sh-cache.outputs.cache-hit == 'true' + + - name: Install dependencies from scratch + shell: bash + run: | + FD_AUTO_INSTALL_PACKAGES=1 '${{ inputs.deps-script-path }}' install + '${{ inputs.deps-bundle-path }}' + if: steps.deps-sh-cache.outputs.cache-hit != 'true' diff --git a/.github/actions/hugepages/action.yml b/.github/actions/hugepages/action.yml new file mode 100644 index 0000000..fe4e086 --- /dev/null +++ b/.github/actions/hugepages/action.yml @@ -0,0 +1,18 @@ +name: hugepages +description: 'Setup 1 GiB gigantic pages' +inputs: + count: + description: 'Number of huge pages' + required: true + default: '64' +outputs: {} +runs: + using: composite + steps: + - shell: bash + run: | + set -x + sudo ./firedancer/src/util/shmem/fd_shmem_cfg fini || true + sudo ./firedancer/src/util/shmem/fd_shmem_cfg init 0666 $USER "" || true + sudo ./firedancer/src/util/shmem/fd_shmem_cfg alloc '${{ inputs.count }}' gigantic 0 + sudo chown -R $USER:$USER /mnt/.fd diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml new file mode 100644 index 0000000..48132e5 --- /dev/null +++ b/.github/workflows/build_and_push.yml @@ -0,0 +1,42 @@ +name: Industry +on: + workflow_call: + workflow_dispatch: +jobs: + publish: + name: Publish Industry Targets + strategy: + matrix: + feature_set: [modern] + runs-on: + ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + # doesnt work locally + # - uses: ./.github/actions/hugepages + # - uses: ./.github/actions/deps + - run: sudo apt update && sudo apt install -y zip + + - name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.FUZZ_SERVICE_ACCT_JSON_BUNDLE }} + + - uses: firedancer-io/fuzzbot-builder@main + name: Build Solana Target + with: + command: cd firedancer && make -j -Otarget shared + + - uses: firedancer-io/clusterfuzz-action@main + name: Upload fuzz targets to ClusterFuzz + with: + bucket-name: $CF_BUCKET + artifact-dir: build/linux/clang/combi/${{ matrix.feature_set }}/fuzz-test + object-prefix: industry-labs/ + project-id: isol-clusterfuzz + qualifier: ${{ matrix.feature_set }} + service-account-credentials: ${{ secrets.FUZZ_SERVICE_ACCT_JSON_BUNDLE }} + + - run: gcloud storage cp build/linux/clang/icelake/industry-bundle.zip gs://firedancer-builds.isol-clusterfuzz.appspot.com/industry-fd/industry-fd.${{ github.sha }}.zip \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..30bd623 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env + diff --git a/firedancer b/firedancer index 266734b..86a6383 160000 --- a/firedancer +++ b/firedancer @@ -1 +1 @@ -Subproject commit 266734b3354080724b3cb8140977743b692318bd +Subproject commit 86a6383f9845b73ea2c92cfce90ba4ac33581a12 diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/rust/diff-sbpf_loader/Cargo.toml b/rust/diff-sbpf_loader/Cargo.toml new file mode 100644 index 0000000..f8e16df --- /dev/null +++ b/rust/diff-sbpf_loader/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "diff-sbpf_loader" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type=["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +solana-bpf-loader-program = { git = "https://github.com/solana-labs/solana", rev = "ded307adb328a34a56a66b5a04eb662fd590484a", version = "1.17.0" } +solana-logger = { git = "https://github.com/solana-labs/solana", rev = "ded307adb328a34a56a66b5a04eb662fd590484a", version = "1.17.0" } +solana-metrics = { git = "https://github.com/solana-labs/solana", rev = "ded307adb328a34a56a66b5a04eb662fd590484a", version = "1.17.0" } +solana-program-runtime = { git = "https://github.com/solana-labs/solana", rev = "ded307adb328a34a56a66b5a04eb662fd590484a", version = "1.17.0" } +solana-sdk = { git = "https://github.com/solana-labs/solana", rev = "ded307adb328a34a56a66b5a04eb662fd590484a", version = "1.17.0" } diff --git a/rust/diff-sbpf_loader/build.rs b/rust/diff-sbpf_loader/build.rs new file mode 100644 index 0000000..f8ef7c8 --- /dev/null +++ b/rust/diff-sbpf_loader/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:-C llvm-args='-sanitizer-coverage-level=4'"); +} diff --git a/rust/diff-sbpf_loader/src/lib.rs b/rust/diff-sbpf_loader/src/lib.rs new file mode 100644 index 0000000..36e7932 --- /dev/null +++ b/rust/diff-sbpf_loader/src/lib.rs @@ -0,0 +1,148 @@ +use solana_program_runtime::compute_budget::ComputeBudget; +use solana_program_runtime::invoke_context::InvokeContext; +use solana_program_runtime::solana_rbpf::elf::Executable; + +use std::sync::Arc; +use std::mem; + +use std::fmt::Debug; + +#[derive(PartialEq)] +pub struct LoadedProgram { + pub rodata: Vec, + pub entry_pc: u64, + pub text_off: i64, + pub text_sz: u64, +} + +impl Debug for LoadedProgram { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + writeln!(f, "Rodata size: 0x{:x}", self.rodata.len())?; + writeln!(f, "Entrypoint: 0x{:x}", self.entry_pc)?; + writeln!(f, "Text offset: 0x{:x}", self.text_off)?; + writeln!(f, "Text size: 0x{:x}", self.text_sz)?; + Ok(()) + } +} + + +#[no_mangle] +pub extern "C" fn industry_init() { + println!("init rust") +} + +#[no_mangle] +extern "C" fn industry_test_one( + out_result_sz: *mut usize, + out_result_buf: *mut u8, + result_buf_sz: usize, + data: *const u8, + data_sz: usize, +) -> i32 { + + // The layout of result (out_result_buf) is: + // - i32: 0 if unpack succeded, -1 if failed + // - u64: len of rodata: + // - uchar[]: rodata + // - u64: entry_pc + // - i64: text_off + // - u64: text_sz + + + let elf = unsafe { + if data.is_null() { + let buffer = std::slice::from_raw_parts_mut(out_result_buf, result_buf_sz); + buffer[..mem::size_of::()].copy_from_slice(&(-1 as i32).to_ne_bytes()); + *out_result_sz = mem::size_of::(); + return 0 + } + + std::slice::from_raw_parts(data, data_sz) + }; + + let mres = load_program_labs(elf); + + if let Err(_e) = mres { + unsafe { + let buffer = std::slice::from_raw_parts_mut(out_result_buf, result_buf_sz); + buffer[..mem::size_of::()].copy_from_slice(&(-1 as i32).to_ne_bytes()); + *out_result_sz = mem::size_of::(); + return 0 + } + } + + let res = mres.unwrap(); + unsafe { + *out_result_sz = 0; + let mut last_sz; + + let mut buffer = std::slice::from_raw_parts_mut(out_result_buf, result_buf_sz); + + // - i32: 0 if unpack succeded, -1 if failed + last_sz = mem::size_of::(); + buffer[..last_sz].copy_from_slice(&(0 as i32).to_ne_bytes()); + buffer = &mut buffer[last_sz..]; + *out_result_sz += last_sz; + + // - u64: len of rodata + last_sz = mem::size_of::(); + buffer[..last_sz].copy_from_slice(&(res.rodata.len()).to_ne_bytes()); + buffer = &mut buffer[last_sz..]; + *out_result_sz += last_sz; + + // - uchar[]: rodata + last_sz = res.rodata.len(); + buffer[..last_sz].copy_from_slice(&res.rodata); + buffer = &mut buffer[last_sz..]; + *out_result_sz += last_sz; + + // - u64: entry_pc + last_sz = mem::size_of::(); + buffer[..last_sz].copy_from_slice(&(res.entry_pc).to_ne_bytes()); + buffer = &mut buffer[last_sz..]; + *out_result_sz += last_sz; + + // - i64: text_off + last_sz = mem::size_of::(); + buffer[..last_sz].copy_from_slice(&(res.text_off).to_ne_bytes()); + buffer = &mut buffer[last_sz..]; + *out_result_sz += last_sz; + + // - u64: text_sz + last_sz = mem::size_of::(); + buffer[..last_sz].copy_from_slice(&(res.text_sz).to_ne_bytes()); + // buffer = &mut buffer[last_sz..]; + *out_result_sz += last_sz; + + 0 + } +} + + +fn load_program_labs(elf: &[u8]) -> Result { + let feature_set = solana_sdk::feature_set::FeatureSet::all_enabled(); + let compute_budget = ComputeBudget::default(); + + let loader = solana_bpf_loader_program::syscalls::create_program_runtime_environment_v1( + &feature_set, + &compute_budget, + // reject_deployment_of_broken_elfs + true, + // debugging_features + false, + ) + .map_err(|e| format!("{:?}", e))?; + let loader = Arc::new(loader); + + let executable: Executable> = + Executable::load(elf, loader).map_err(|e| format!("Labs load err: {:?}", e))?; + + let ro_section = executable.get_ro_section().to_vec(); + let (text_vaddr, text_section) = executable.get_text_bytes(); + Ok(LoadedProgram { + rodata: ro_section, + entry_pc: executable.get_entrypoint_instruction_offset() as u64, + text_off: (text_vaddr - 0x1_0000_0000) as i64, + text_sz: (text_section.len() as u64) & (!7u64), // not necessarily multiple of 8 + }) +} From 61783301c7e95cec54c8c839ebaa2a75ec797d64 Mon Sep 17 00:00:00 2001 From: nlgripto <155116201+nlgripto@users.noreply.github.com> Date: Thu, 4 Jan 2024 21:01:57 +0000 Subject: [PATCH 3/3] remove submod, update workflow --- .github/workflows/build_and_push.yml | 19 +++++++------------ .gitmodules | 3 --- firedancer | 1 - 3 files changed, 7 insertions(+), 16 deletions(-) delete mode 100644 .gitmodules delete mode 160000 firedancer diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 48132e5..9d3ccb9 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -17,7 +17,6 @@ jobs: # doesnt work locally # - uses: ./.github/actions/hugepages # - uses: ./.github/actions/deps - - run: sudo apt update && sudo apt install -y zip - name: 'Authenticate to Google Cloud' uses: 'google-github-actions/auth@v2' @@ -27,16 +26,12 @@ jobs: - uses: firedancer-io/fuzzbot-builder@main name: Build Solana Target with: - command: cd firedancer && make -j -Otarget shared + command: apt update && apt install -y zip && pushd rust/diff-sbpf_loader && cargo build && + mkdir ../../artdir && popd && zip ./labs-industry-bundle.zip rust/target/debug/*.so - - uses: firedancer-io/clusterfuzz-action@main - name: Upload fuzz targets to ClusterFuzz + - id: 'upload-file' + uses: 'google-github-actions/upload-cloud-storage@v2' with: - bucket-name: $CF_BUCKET - artifact-dir: build/linux/clang/combi/${{ matrix.feature_set }}/fuzz-test - object-prefix: industry-labs/ - project-id: isol-clusterfuzz - qualifier: ${{ matrix.feature_set }} - service-account-credentials: ${{ secrets.FUZZ_SERVICE_ACCT_JSON_BUNDLE }} - - - run: gcloud storage cp build/linux/clang/icelake/industry-bundle.zip gs://firedancer-builds.isol-clusterfuzz.appspot.com/industry-fd/industry-fd.${{ github.sha }}.zip \ No newline at end of file + path: './labs-industry-bundle.zip' + destination: firedancer-builds.isol-clusterfuzz.appspot.com/industry-labs/industry-labs.${{ github.sha }}.zip + # - run: gcloud storage cp ./labs-industry-bundle.zip $CF_BUCKET/industry-labs/industry-labs.${{ github.sha }}.zip \ No newline at end of file diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 7a37d8a..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "firedancer"] - path = firedancer - url = git@github.com:firedancer-io/firedancer.git diff --git a/firedancer b/firedancer deleted file mode 160000 index 86a6383..0000000 --- a/firedancer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 86a6383f9845b73ea2c92cfce90ba4ac33581a12