diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle index bdce4929d95e..a65592026303 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst.isle +++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle @@ -21,6 +21,13 @@ ;; A pseudo-instruction to update unwind info. (Unwind (inst UnwindInst)) + ;; Implementation of `br_table`, uses `idx` to jump to one of `targets` or + ;; jumps to `default` is it's out-of-bounds. + (BrTable + (idx XReg) + (default MachLabel) + (targets BoxVecMachLabel)) + ;;;; Actual Instructions ;;;; ;; Raise a trap. @@ -547,6 +554,10 @@ (_ Unit (emit (MInst.BitcastIntFromFloat64 dst src)))) dst)) +(decl gen_br_table (XReg MachLabel BoxVecMachLabel) Unit) +(rule (gen_br_table idx default labels) + (emit (MInst.BrTable idx default labels))) + ;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput) diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index 3772c127955b..af593a2708e8 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -104,8 +104,8 @@ where // (with an `EmitIsland`). We check this in debug builds. This is `mut` // to allow disabling the check for `JTSequence`, which is always // emitted following an `EmitIsland`. - let start = sink.cur_offset(); - pulley_emit(self, sink, emit_info, state, start); + let mut start = sink.cur_offset(); + pulley_emit(self, sink, emit_info, state, &mut start); let end = sink.cur_offset(); assert!( @@ -124,9 +124,9 @@ where fn pulley_emit

( inst: &Inst, sink: &mut MachBuffer>, - _emit_info: &EmitInfo, + emit_info: &EmitInfo, state: &mut EmitState

, - start_offset: u32, + start_offset: &mut u32, ) where P: PulleyTargetKind, { @@ -218,8 +218,8 @@ fn pulley_emit

( Inst::IndirectCall { .. } => todo!(), Inst::Jump { label } => { - sink.use_label_at_offset(start_offset + 1, *label, LabelUse::Jump(1)); - sink.add_uncond_branch(start_offset, start_offset + 5, *label); + sink.use_label_at_offset(*start_offset + 1, *label, LabelUse::Jump(1)); + sink.add_uncond_branch(*start_offset, *start_offset + 5, *label); enc::jump(sink, 0x00000000); } @@ -229,7 +229,7 @@ fn pulley_emit

( not_taken, } => { // If taken. - let taken_start = start_offset + 2; + let taken_start = *start_offset + 2; let taken_end = taken_start + 4; sink.use_label_at_offset(taken_start, *taken, LabelUse::Jump(2)); @@ -237,10 +237,10 @@ fn pulley_emit

( enc::br_if_not(&mut inverted, c, 0x00000000); debug_assert_eq!( inverted.len(), - usize::try_from(taken_end - start_offset).unwrap() + usize::try_from(taken_end - *start_offset).unwrap() ); - sink.add_cond_branch(start_offset, taken_end, *taken, &inverted); + sink.add_cond_branch(*start_offset, taken_end, *taken, &inverted); enc::br_if(sink, c, 0x00000000); debug_assert_eq!(sink.cur_offset(), taken_end); @@ -261,7 +261,7 @@ fn pulley_emit

( } => { br_if_cond_helper( sink, - start_offset, + *start_offset, *src1, *src2, taken, @@ -279,7 +279,7 @@ fn pulley_emit

( } => { br_if_cond_helper( sink, - start_offset, + *start_offset, *src1, *src2, taken, @@ -297,7 +297,7 @@ fn pulley_emit

( } => { br_if_cond_helper( sink, - start_offset, + *start_offset, *src1, *src2, taken, @@ -315,7 +315,7 @@ fn pulley_emit

( } => { br_if_cond_helper( sink, - start_offset, + *start_offset, *src1, *src2, taken, @@ -333,7 +333,7 @@ fn pulley_emit

( } => { br_if_cond_helper( sink, - start_offset, + *start_offset, *src1, *src2, taken, @@ -351,7 +351,7 @@ fn pulley_emit

( } => { br_if_cond_helper( sink, - start_offset, + *start_offset, *src1, *src2, taken, @@ -484,6 +484,47 @@ fn pulley_emit

( Inst::BitcastIntFromFloat64 { dst, src } => enc::bitcast_int_from_float_64(sink, dst, src), Inst::BitcastFloatFromInt32 { dst, src } => enc::bitcast_float_from_int_32(sink, dst, src), Inst::BitcastFloatFromInt64 { dst, src } => enc::bitcast_float_from_int_64(sink, dst, src), + + Inst::BrTable { + idx, + default, + targets, + } => { + // Encode the `br_table32` instruction directly which expects the + // next `amt` 4-byte integers to all be relative offsets. Each + // offset is the pc-relative offset of the branch destination. + // + // Pulley clamps the branch targets to the `amt` specified so the + // final branch target is the default jump target. + // + // Note that this instruction may have many branch targets so it + // manually checks to see if an island is needed. If so we emit a + // jump around the island before the `br_table32` itself gets + // emitted. + let amt = u32::try_from(targets.len() + 1).expect("too many branch targets"); + let br_table_size = amt * 4 + 6; + if sink.island_needed(br_table_size) { + let label = sink.get_label(); + >::from(Inst::Jump { label }).emit(sink, emit_info, state); + sink.emit_island(br_table_size, &mut state.ctrl_plane); + sink.bind_label(label, &mut state.ctrl_plane); + } + enc::br_table32(sink, *idx, amt); + for target in targets.iter() { + let offset = sink.cur_offset(); + sink.use_label_at_offset(offset, *target, LabelUse::Jump(0)); + sink.put4(0); + } + let offset = sink.cur_offset(); + sink.use_label_at_offset(offset, *default, LabelUse::Jump(0)); + sink.put4(0); + + // We manually handled `emit_island` above when dealing with + // `island_needed` so update the starting offset to the current + // offset so this instruction doesn't accidentally trigger + // the assertion that we're always under worst-case-size. + *start_offset = sink.cur_offset(); + } } } diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs index 0e8d3f346ce4..4d5cb08001bd 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/mod.rs @@ -244,6 +244,10 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) { collector.reg_use(src); collector.reg_def(dst); } + + Inst::BrTable { idx, .. } => { + collector.reg_use(idx); + } } } @@ -370,6 +374,7 @@ where | Inst::BrIfXslteq32 { .. } | Inst::BrIfXult32 { .. } | Inst::BrIfXulteq32 { .. } => MachTerminator::Cond, + Inst::BrTable { .. } => MachTerminator::Indirect, _ => MachTerminator::None, } } @@ -437,8 +442,8 @@ where } } - fn gen_jump(_target: MachLabel) -> Self { - todo!() + fn gen_jump(target: MachLabel) -> Self { + Inst::Jump { label: target }.into() } fn worst_case_size() -> CodeOffset { @@ -839,6 +844,15 @@ impl Inst { let src = format_reg(**src); format!("{dst} = bitcast_float_from_int64 {src}") } + + Inst::BrTable { + idx, + default, + targets, + } => { + let idx = format_reg(**idx); + format!("br_table {idx} {default:?} {targets:?}") + } } } } diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle index 5659ccca3887..b03d3b1e33d5 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower.isle +++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle @@ -49,10 +49,8 @@ (lower_brif_of_icmp32 (IntCC.UnsignedLessThanOrEqual) b a then else)) ;; Branch tables. -(decl lower_br_table (Reg MachLabelSlice) Unit) -(extern constructor lower_br_table lower_br_table) -(rule (lower_branch (br_table index _) targets) - (lower_br_table index targets)) +(rule (lower_branch (br_table index _) (jump_table_targets default targets)) + (gen_br_table index default targets)) ;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs b/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs index e1c57bd73bc1..f75fe027f239 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs +++ b/cranelift/codegen/src/isa/pulley_shared/lower/isle.rs @@ -51,10 +51,6 @@ where crate::isle_lower_prelude_methods!(InstAndKind

); crate::isle_prelude_caller_methods!(PulleyABICallSite

); - fn lower_br_table(&mut self, _index: Reg, _targets: &[MachLabel]) -> Unit { - todo!() - } - fn vreg_new(&mut self, r: Reg) -> VReg { VReg::new(r).unwrap() } diff --git a/cranelift/filetests/filetests/isa/pulley32/br_table.clif b/cranelift/filetests/filetests/isa/pulley32/br_table.clif new file mode 100644 index 000000000000..624b808192c1 --- /dev/null +++ b/cranelift/filetests/filetests/isa/pulley32/br_table.clif @@ -0,0 +1,69 @@ +test compile precise-output +target pulley32 + +function %br_table(i32) -> i32 { +block0(v0: i32): + br_table v0, block4, [block1, block2, block2, block3] + +block1: + v1 = iconst.i32 1 + jump block5(v1) + +block2: + v2 = iconst.i32 2 + jump block5(v2) + +block3: + v3 = iconst.i32 3 + jump block5(v3) + +block4: + v4 = iconst.i32 4 + jump block5(v4) + +block5(v5: i32): + v6 = iadd.i32 v0, v5 + return v6 +} + +; VCode: +; block0: +; br_table x0 MachLabel(6) [MachLabel(5), MachLabel(1), MachLabel(2), MachLabel(3)] +; block1: +; jump label4 +; block2: +; jump label4 +; block3: +; x5 = xconst8 3 +; jump label7 +; block4: +; x5 = xconst8 2 +; jump label7 +; block5: +; x5 = xconst8 1 +; jump label7 +; block6: +; x5 = xconst8 4 +; jump label7 +; block7: +; x0 = xadd32 x0, x5 +; ret +; +; Disassembled: +; br_table32 x0, 5 +; 0x29 // target = 0x2f +; 0x1d // target = 0x27 +; 0x19 // target = 0x27 +; 0xd // target = 0x1f +; 0x21 // target = 0x37 +; jump 0xd // target = 0x27 +; xconst8 x5, 3 +; jump 0x18 // target = 0x3a +; xconst8 x5, 2 +; jump 0x10 // target = 0x3a +; xconst8 x5, 1 +; jump 0x8 // target = 0x3a +; xconst8 x5, 4 +; xadd32 x0, x0, x5 +; ret + diff --git a/cranelift/filetests/filetests/isa/pulley64/br_table.clif b/cranelift/filetests/filetests/isa/pulley64/br_table.clif new file mode 100644 index 000000000000..8c334abc9be4 --- /dev/null +++ b/cranelift/filetests/filetests/isa/pulley64/br_table.clif @@ -0,0 +1,69 @@ +test compile precise-output +target pulley64 + +function %br_table(i32) -> i32 { +block0(v0: i32): + br_table v0, block4, [block1, block2, block2, block3] + +block1: + v1 = iconst.i32 1 + jump block5(v1) + +block2: + v2 = iconst.i32 2 + jump block5(v2) + +block3: + v3 = iconst.i32 3 + jump block5(v3) + +block4: + v4 = iconst.i32 4 + jump block5(v4) + +block5(v5: i32): + v6 = iadd.i32 v0, v5 + return v6 +} + +; VCode: +; block0: +; br_table x0 MachLabel(6) [MachLabel(5), MachLabel(1), MachLabel(2), MachLabel(3)] +; block1: +; jump label4 +; block2: +; jump label4 +; block3: +; x5 = xconst8 3 +; jump label7 +; block4: +; x5 = xconst8 2 +; jump label7 +; block5: +; x5 = xconst8 1 +; jump label7 +; block6: +; x5 = xconst8 4 +; jump label7 +; block7: +; x0 = xadd32 x0, x5 +; ret +; +; Disassembled: +; br_table32 x0, 5 +; 0x29 // target = 0x2f +; 0x1d // target = 0x27 +; 0x19 // target = 0x27 +; 0xd // target = 0x1f +; 0x21 // target = 0x37 +; jump 0xd // target = 0x27 +; xconst8 x5, 3 +; jump 0x18 // target = 0x3a +; xconst8 x5, 2 +; jump 0x10 // target = 0x3a +; xconst8 x5, 1 +; jump 0x8 // target = 0x3a +; xconst8 x5, 4 +; xadd32 x0, x0, x5 +; ret + diff --git a/pulley/fuzz/src/interp.rs b/pulley/fuzz/src/interp.rs index 5fd70b082c16..e0bac2e104fd 100644 --- a/pulley/fuzz/src/interp.rs +++ b/pulley/fuzz/src/interp.rs @@ -116,6 +116,7 @@ fn op_is_safe_for_fuzzing(op: &Op) -> bool { Op::XPop32(_) | Op::XPop64(_) => false, Op::XPush32Many(_) | Op::XPush64Many(_) => false, Op::XPop32Many(_) | Op::XPop64Many(_) => false, + Op::BrTable32(_) => false, } } diff --git a/pulley/src/disas.rs b/pulley/src/disas.rs index 248309cd45d1..b146773660df 100644 --- a/pulley/src/disas.rs +++ b/pulley/src/disas.rs @@ -88,6 +88,17 @@ impl<'a> Disassembler<'a> { val.disas(self.start + self.start_offset, &mut self.temp); } } + + fn disas_br_table32(&mut self, reg: XReg, amt: u32) { + self.disas_op("br_table32", &[®, &amt]); + for _ in 0..amt { + self.after_visit(); + self.start = self.bytecode.position(); + if let Ok(offset) = PcRelOffset::decode(self.bytecode()) { + offset.disas(self.start, &mut self.temp); + } + } + } } /// Anything inside an instruction that can be disassembled: registers, @@ -206,11 +217,36 @@ macro_rules! impl_disas { )* ) => { $( - fn $snake_name(&mut self $( $( , $field : $field_ty )* )? ) { - self.disas_op(stringify!($snake_name), &[$($(&$field),*)?]) - } + impl_disas!(@one $snake_name = $name $( { $($field: $field_ty),* } )?); )* }; + + // Diassembling `br_table` is a bit special as it has trailing byte after + // the opcode of the branch table itself. + ( + @one br_table32 = BrTable32 $( { + $( + $field:ident : $field_ty:ty + ),* + } )? + ) => { + fn br_table32(&mut self $( $( , $field : $field_ty )* )? ) { + self.disas_br_table32($($($field),*)?) + } + }; + + // All other opcodes other than `br_table` are handled in the same manner. + ( + @one $snake_name:ident = $name:ident $( { + $( + $field:ident : $field_ty:ty + ),* + } )? + ) => { + fn $snake_name(&mut self $( $( , $field : $field_ty )* )? ) { + self.disas_op(stringify!($snake_name), &[$($(&$field),*)?]) + } + }; } impl<'a> OpVisitor for Disassembler<'a> { diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs index c0817f454144..51200c79cc05 100644 --- a/pulley/src/interp.rs +++ b/pulley/src/interp.rs @@ -1161,6 +1161,15 @@ impl OpVisitor for Interpreter<'_> { self.state[dst].set_f64(f64::from_ne_bytes(val.to_ne_bytes())); ControlFlow::Continue(()) } + + fn br_table32(&mut self, idx: XReg, amt: u32) -> ControlFlow { + let idx = self.state[idx].get_u32().min(amt - 1) as isize; + // SAFETY: part of the contract of the interpreter is only dealing with + // valid bytecode, so this offset should be safe. + self.pc = unsafe { self.pc.offset(idx * 4) }; + let rel = unwrap_uninhabited(PcRelOffset::decode(&mut self.pc)); + self.pc_rel_jump(rel, 0) + } } impl ExtendedOpVisitor for Interpreter<'_> { diff --git a/pulley/src/lib.rs b/pulley/src/lib.rs index b80a5646ec9b..e039138a85c8 100644 --- a/pulley/src/lib.rs +++ b/pulley/src/lib.rs @@ -60,6 +60,14 @@ macro_rules! for_each_op { /// Branch if unsigned `a <= b`. br_if_xulteq64 = BrIfXulteq64 { a: XReg, b: XReg, offset: PcRelOffset }; + /// Branch to the label indicated by `idx`. + /// + /// After this instruction are `amt` instances of `PcRelOffset` + /// and the `idx` selects which one will be branched to. The value + /// of `idx` is clamped to `amt - 1` (e.g. the last offset is the + /// "default" one. + br_table32 = BrTable32 { idx: XReg, amt: u32 }; + /// Move between `x` registers. xmov = Xmov { dst: XReg, src: XReg }; /// Move between `f` registers. diff --git a/pulley/tests/all/disas.rs b/pulley/tests/all/disas.rs index 68b3ff06e281..ca5f9fc93667 100644 --- a/pulley/tests/all/disas.rs +++ b/pulley/tests/all/disas.rs @@ -156,3 +156,32 @@ ret "#, ); } + +#[test] +fn disassemble_br_table() { + let mut bytecode = Vec::new(); + PushFrame {}.encode(&mut bytecode); + BrTable32 { + idx: XReg::x1, + amt: 4, + } + .encode(&mut bytecode); + bytecode.extend_from_slice(&0_i32.to_le_bytes()); + bytecode.extend_from_slice(&0_i32.to_le_bytes()); + bytecode.extend_from_slice(&0_i32.to_le_bytes()); + bytecode.extend_from_slice(&0_i32.to_le_bytes()); + PopFrame {}.encode(&mut bytecode); + + assert_disas_with_disassembler( + &mut disas::Disassembler::new(&bytecode), + r#" + 0: 35 push_frame + 1: 43 01 04 00 00 00 br_table32 x1, 4 + 7: 00 00 00 00 0x0 // target = 0x7 + b: 00 00 00 00 0x0 // target = 0xb + f: 00 00 00 00 0x0 // target = 0xf + 13: 00 00 00 00 0x0 // target = 0x13 + 17: 36 pop_frame + "#, + ); +} diff --git a/tests/disas.rs b/tests/disas.rs index 74baea36221d..7f1268d7ef81 100644 --- a/tests/disas.rs +++ b/tests/disas.rs @@ -358,7 +358,7 @@ fn assert_output( disas.offsets(false); disas.hexdump(false); let mut decoder = pulley_interpreter::decode::Decoder::new(); - + let mut last_disas_pos = 0; loop { let addr = disas.bytecode().position(); @@ -375,22 +375,17 @@ fn assert_output( } Ok(()) => { - let disassembly = disas - .disas() - .lines() - .map(|l| l.trim().to_string()) - .filter(|l| !l.is_empty()) - .next_back() - .unwrap(); + let disassembly = disas.disas()[last_disas_pos..].trim(); + last_disas_pos = disas.disas().len(); let address = u64::try_from(addr).unwrap(); let is_jump = - disassembly.contains("jump") || disassembly.contains("br_if"); + disassembly.contains("jump") || disassembly.contains("br_"); let is_return = disassembly == "ret"; result.push(DisasInst { address, is_jump, is_return, - disassembly, + disassembly: disassembly.to_string(), }); } } @@ -459,14 +454,15 @@ where disassembly: disas, } in disas(bytes, sym.address())?.into_iter() { - if write_offsets || (prev_jump && !is_jump) { - write!(result, "{address:>4x}: ")?; - } else { - write!(result, " ")?; + for (i, line) in disas.lines().enumerate() { + if i == 0 && (write_offsets || (prev_jump && !is_jump)) { + write!(result, "{address:>4x}: ")?; + } else { + write!(result, " ")?; + } + writeln!(result, "{line}")?; } - writeln!(result, "{disas}")?; - prev_jump = is_jump; // Flip write_offsets to true once we've seen a `ret`, as diff --git a/tests/disas/pulley/br_table.wat b/tests/disas/pulley/br_table.wat new file mode 100644 index 000000000000..63ec9cefde36 --- /dev/null +++ b/tests/disas/pulley/br_table.wat @@ -0,0 +1,48 @@ +;;! target = "pulley64" +;;! test = "compile" + +(module + (func (param i32) (result i32) + block $a + block $b + block $c + local.get 0 + br_table $a $b $c + end + i32.const 0 + return + end + i32.const 1 + return + end + i32.const 2 + ) +) +;; wasm[0]::function[0]: +;; xconst8 spilltmp0, -16 +;; xadd32 sp, sp, spilltmp0 +;; store64_offset8 sp, 8, lr +;; store64 sp, fp +;; xmov fp, sp +;; br_table32 x2, 3 +;; 0x1d // target = 0x33 +;; 0x8 // target = 0x22 +;; 0x26 // target = 0x44 +;; 22: xconst8 x0, 1 +;; load64_offset8 lr, sp, 8 +;; load64 fp, sp +;; xconst8 spilltmp0, 16 +;; xadd32 sp, sp, spilltmp0 +;; ret +;; 33: xconst8 x0, 2 +;; 36: load64_offset8 lr, sp, 8 +;; 3a: load64 fp, sp +;; 3d: xconst8 spilltmp0, 16 +;; 40: xadd32 sp, sp, spilltmp0 +;; 43: ret +;; 44: xconst8 x0, 0 +;; 47: load64_offset8 lr, sp, 8 +;; 4b: load64 fp, sp +;; 4e: xconst8 spilltmp0, 16 +;; 51: xadd32 sp, sp, spilltmp0 +;; 54: ret