Skip to content

Commit

Permalink
Add tests and calibration for new VM instantiation cost model
Browse files Browse the repository at this point in the history
  • Loading branch information
@graydon
graydon committed Mar 21, 2024
1 parent 94165df commit bf8bcf2
Show file tree
Hide file tree
Showing 8 changed files with 665 additions and 108 deletions.
79 changes: 51 additions & 28 deletions soroban-env-host/benches/common/cost_types/wasm_insn_exec.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,56 @@ struct WasmModule {
overhead: u64,
}

// ModEmitter's default constructors are a little too spartan for our needs, we
// want our benchmarks to all have at least one imported function and at least
// one defined and exported function, so we're in the right performance tier.
// But we also don't want to go changing those constructors since it'll perturb
// a lot of non-benchmark users.
trait ModEmitterExt {
fn bench_default() -> Self;
fn bench_from_configs(mem_pages: u32, elem_count: u32) -> Self;
fn add_bench_import(self) -> Self;
fn add_bench_export(self) -> Self;
fn add_bench_baseline_material(self) -> Self;
}

impl ModEmitterExt for ModEmitter {
fn add_bench_import(mut self) -> Self {
self.import_func("t", "_", Arity(0));
self
}
fn add_bench_export(self) -> Self {
let mut fe = self.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
fe.finish_and_export("default")
}
fn add_bench_baseline_material(self) -> Self {
self.add_bench_import().add_bench_export()
}

fn bench_default() -> Self {
Self::add_bench_baseline_material(ModEmitter::default())
}

fn bench_from_configs(mem_pages: u32, elem_count: u32) -> Self {
Self::add_bench_baseline_material(ModEmitter::from_configs(mem_pages, elem_count))
}
}

pub fn wasm_module_with_n_internal_funcs(n: usize) -> Vec<u8> {
let mut me = ModEmitter::default();
let mut me = ModEmitter::bench_default();
for _ in 0..n {
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
(me, _) = fe.finish();
}
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
fe.finish_and_export("test").finish()
me.finish()
}

pub fn wasm_module_with_n_insns(n: usize) -> Vec<u8> {
// We actually emit 4 instructions per loop iteration, so we need to divide by 4.
let n = 1 + (n / 4);
let mut fe = ModEmitter::default().func(Arity(1), 0);
let mut fe = ModEmitter::bench_default().func(Arity(1), 0);
let arg = fe.args[0];
fe.push(Operand::Const64(1));
for i in 0..n {
Expand All @@ -43,31 +77,27 @@ pub fn wasm_module_with_n_insns(n: usize) -> Vec<u8> {
fe.finish_and_export("test").finish()
}
pub fn wasm_module_with_n_globals(n: usize) -> Vec<u8> {
let mut me = ModEmitter::default();
let mut me = ModEmitter::bench_default();
for i in 0..n {
me.global(ValType::I64, true, &ConstExpr::i64_const(i as i64));
}
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
fe.finish_and_export("test").finish()
me.finish()
}

pub fn wasm_module_with_n_imports(n: usize) -> Vec<u8> {
let mut me = ModEmitter::default();
let mut me = ModEmitter::default().add_bench_import();
let names = Vm::get_all_host_functions();
for (module, name, arity) in names.iter().take(n) {
if *module == "t" {
continue;
}
me.import_func(module, name, Arity(*arity));
}
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
fe.finish_and_export("test").finish()
me.add_bench_export().finish()
}

pub fn wasm_module_with_n_exports(n: usize) -> Vec<u8> {
let me = ModEmitter::default();
let me = ModEmitter::bench_default();
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
let (mut me, fid) = fe.finish();
Expand All @@ -78,7 +108,7 @@ pub fn wasm_module_with_n_exports(n: usize) -> Vec<u8> {
}

pub fn wasm_module_with_n_table_entries(n: usize) -> Vec<u8> {
let me = ModEmitter::from_configs(1, n as u32);
let me = ModEmitter::bench_from_configs(1, n as u32);
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
let (mut me, f) = fe.finish();
Expand All @@ -88,7 +118,7 @@ pub fn wasm_module_with_n_table_entries(n: usize) -> Vec<u8> {
}

pub fn wasm_module_with_n_types(mut n: usize) -> Vec<u8> {
let mut me = ModEmitter::default();
let mut me = ModEmitter::bench_default();
// There's a max of 1,000,000 types, so we just make a loop
// that covers more than that many combinations, and break when we've got
// to the requested number.
Expand Down Expand Up @@ -151,13 +181,11 @@ pub fn wasm_module_with_n_types(mut n: usize) -> Vec<u8> {
}
}
}
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
fe.finish_and_export("test").finish()
me.finish()
}

pub fn wasm_module_with_n_elem_segments(n: usize) -> Vec<u8> {
let me = ModEmitter::from_configs(1, n as u32);
let me = ModEmitter::bench_from_configs(1, n as u32);
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
let (mut me, f) = fe.finish();
Expand All @@ -169,22 +197,17 @@ pub fn wasm_module_with_n_elem_segments(n: usize) -> Vec<u8> {

pub fn wasm_module_with_n_data_segments(n: usize) -> Vec<u8> {
let mem_offset = n as u32 * 1024;
let me = ModEmitter::from_configs(1 + mem_offset / 65536, 0);
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
let (mut me, _) = fe.finish();
let mut me = ModEmitter::bench_from_configs(1 + mem_offset / 65536, 0);
for _ in 0..n {
me.define_data_segment(n as u32 * 1024, vec![1, 2, 3, 4]);
}
me.finish()
}

pub fn wasm_module_with_n_memory_pages(n: usize) -> Vec<u8> {
let mut me = ModEmitter::from_configs(n as u32, 0);
let mut me = ModEmitter::bench_from_configs(n as u32, 0);
me.define_data_segment(0, vec![0xff; n * 0x10000]);
let mut fe = me.func(Arity(0), 0);
fe.push(Symbol::try_from_small_str("pass").unwrap());
fe.finish_and_export("test").finish()
me.finish()
}

fn wasm_module_with_mem_grow(n_pages: usize) -> Vec<u8> {
Expand Down
91 changes: 89 additions & 2 deletions soroban-env-host/benches/worst_case_linear_models.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,87 @@ fn write_cost_params_table<T: Display>(
tw.flush()
}

fn correct_multi_variable_models(
params: &mut BTreeMap<CostType, (MeteredCostComponent, MeteredCostComponent)>,
) {
// Several cost types actually represent additional terms a cost model that
// we're decomposing into multiple variables, such as the cost of VM
// instantiation. When we charge these costs, we charge each variable
// separately, i.e. to charge a 5-variable cost we'll make 5 calls to the
// budget. Only the first of these 5 calls should have a constant factor,
// the rest should have zero as their constant (since they only contribute a
// new linear term), but the calibration code will have put the same (or
// nearly-the-same) nonzero constant term in each `CostComponent`. We
// correct this here by zeroing out the constant term in all but the first
// `CostComponent` of each set, (and attempting to confirm that they all
// have roughly-the-same constant term).
use ContractCostType::*;
const MULTI_VARIABLE_COST_GROUPS: &[&[ContractCostType]] = &[
&[
ParseWasmInstructions,
ParseWasmFunctions,
ParseWasmGlobals,
ParseWasmTableEntries,
ParseWasmTypes,
ParseWasmDataSegments,
ParseWasmElemSegments,
ParseWasmImports,
ParseWasmExports,
ParseWasmMemoryPages,
],
&[
InstantiateWasmInstructions,
InstantiateWasmFunctions,
InstantiateWasmGlobals,
InstantiateWasmTableEntries,
InstantiateWasmTypes,
InstantiateWasmDataSegments,
InstantiateWasmElemSegments,
InstantiateWasmImports,
InstantiateWasmExports,
InstantiateWasmMemoryPages,
],
];
for group in MULTI_VARIABLE_COST_GROUPS {
let mut iter = group.iter();
if let Some(first) = iter.next() {
let Some((first_cpu, first_mem)) = params.get(&CostType::Contract(*first)).cloned()
else {
continue;
};
for ty in iter {
let Some((cpu, mem)) = params.get_mut(&CostType::Contract(*ty)) else {
continue;
};
let cpu_const_diff_ratio = (cpu.const_term as f64 - first_cpu.const_term as f64)
/ first_cpu.const_term as f64;
let mem_const_diff_ratio = (mem.const_term as f64 - first_mem.const_term as f64)
/ first_mem.const_term as f64;
assert!(
cpu_const_diff_ratio < 0.25,
"cost type {:?} has too large a constant CPU term over {:?}: {:?} vs. {:?} ({:?} diff)",
ty,
first,
cpu.const_term,
first_cpu.const_term,
cpu_const_diff_ratio
);
assert!(
mem_const_diff_ratio < 0.25,
"cost type {:?} has too large a constant memory term over {:?}: {:?} vs. {:?} ({:?} diff)",
ty,
first,
mem.const_term,
first_mem.const_term,
mem_const_diff_ratio
);
cpu.const_term = 0;
mem.const_term = 0;
}
}
}
}

fn write_budget_params_code(
params: &BTreeMap<CostType, (MeteredCostComponent, MeteredCostComponent)>,
wasm_tier_cost: &BTreeMap<WasmInsnTier, u64>,
Expand Down Expand Up @@ -335,12 +416,18 @@ fn extract_wasmi_fuel_costs(

#[cfg(all(test, any(target_os = "linux", target_os = "macos")))]
fn main() -> std::io::Result<()> {
let params = if std::env::var("RUN_EXPERIMENT").is_err() {
let mut params = if std::env::var("RUN_EXPERIMENT").is_err() {
for_each_host_cost_measurement::<WorstCaseLinearModels>()?
} else {
for_each_experimental_cost_measurement::<WorstCaseLinearModels>()?
};
let params_wasm = for_each_wasm_insn_measurement::<WorstCaseLinearModels>()?;
let params_wasm = if std::env::var("SKIP_WASM_INSNS").is_err() {
BTreeMap::new()
} else {
for_each_wasm_insn_measurement::<WorstCaseLinearModels>()?
};

correct_multi_variable_models(&mut params);

let mut tw = TabWriter::new(vec![])
.padding(5)
Expand Down
Loading

0 comments on commit bf8bcf2

Please sign in to comment.