Switch to using Wasmtime-style builtins for ceil, floor, etc. (#10657)

* Switch to using Wasmtime-style builtins for ceil, floor, etc.

With this patch, all emitted calls are Wasmtime-style builtins, rather
than Cranelift-style libcalls. This ensures that all calls from
Cranelift-generated code into Wasmtime host code use the same mechanism,
and eliminates the relocation handling code for the libcall mechanism.

* Update tests.

* Avoid using x86_pshufb on non-x86 platforms.

* Revert unneeded f32/f64 changes in Pulley.

* Define i8x16 as an unconstructible type if sse isn't available.

* Delete the setters too.

* Fix f32/f64 setters.

* Test with prtest:full.

prtest:full

* Support fma.

* Return true for `has_native_fma` on pulley.

This works because pulley already has code implementing fma. This avoids
needing to marshal f32x4 values into builtin function calls on pulley.

* Update tests.

Author: sunfishcode

cranelift/codegen/src/isa/aarch64/mod.rs

@@ -223,6 +223,10 @@ impl TargetIsa for AArch64Backend {
         true
     }
 
+    fn has_round(&self) -> bool {
+        true
+    }
+
     fn has_x86_blendv_lowering(&self, _: Type) -> bool {
         false
     }

cranelift/codegen/src/isa/mod.rs

@@ -385,6 +385,9 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
     /// not detected.
     fn has_native_fma(&self) -> bool;
 
+    /// Returns whether this ISA has instructions for `ceil`, `floor`, etc.
+    fn has_round(&self) -> bool;
+
     /// Returns whether the CLIF `x86_blendv` instruction is implemented for
     /// this ISA for the specified type.
     fn has_x86_blendv_lowering(&self, ty: Type) -> bool;

cranelift/codegen/src/isa/pulley_shared/mod.rs

@@ -225,7 +225,13 @@ where
     }
 
     fn has_native_fma(&self) -> bool {
-        false
+        // The pulley interpreter does have fma opcodes.
+        true
+    }
+
+    fn has_round(&self) -> bool {
+        // The pulley interpreter does have rounding opcodes.
+        true
     }
 
     fn has_x86_blendv_lowering(&self, _ty: ir::Type) -> bool {

cranelift/codegen/src/isa/riscv64/mod.rs

@@ -201,6 +201,10 @@ impl TargetIsa for Riscv64Backend {
         true
     }
 
+    fn has_round(&self) -> bool {
+        true
+    }
+
     fn has_x86_blendv_lowering(&self, _: Type) -> bool {
         false
     }

cranelift/codegen/src/isa/s390x/mod.rs

@@ -183,6 +183,10 @@ impl TargetIsa for S390xBackend {
         true
     }
 
+    fn has_round(&self) -> bool {
+        true
+    }
+
     fn has_x86_blendv_lowering(&self, _: Type) -> bool {
         false
     }

cranelift/codegen/src/isa/x64/mod.rs

@@ -167,6 +167,10 @@ impl TargetIsa for X64Backend {
         self.x64_flags.use_fma()
     }
 
+    fn has_round(&self) -> bool {
+        self.x64_flags.use_sse41()
+    }
+
     fn has_x86_blendv_lowering(&self, ty: Type) -> bool {
         // The `blendvpd`, `blendvps`, and `pblendvb` instructions are all only
         // available from SSE 4.1 and onwards. Otherwise the i16x8 type has no

crates/cranelift/src/func_environ.rs

@@ -12,7 +12,7 @@ use cranelift_codegen::ir::immediates::{Imm64, Offset32};
 use cranelift_codegen::ir::pcc::Fact;
 use cranelift_codegen::ir::types::*;
 use cranelift_codegen::ir::{self, types};
-use cranelift_codegen::ir::{ArgumentPurpose, Function, InstBuilder, MemFlags};
+use cranelift_codegen::ir::{ArgumentPurpose, ConstantData, Function, InstBuilder, MemFlags};
 use cranelift_codegen::isa::{TargetFrontendConfig, TargetIsa};
 use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap};
 use cranelift_frontend::FunctionBuilder;
@@ -3219,10 +3219,6 @@ impl FuncEnvironment<'_> {
         self.isa.has_x86_blendv_lowering(ty)
     }
 
-    pub fn use_x86_pshufb_for_relaxed_swizzle(&self) -> bool {
-        self.isa.has_x86_pshufb_lowering()
-    }
-
     pub fn use_x86_pmulhrsw_for_relaxed_q15mul(&self) -> bool {
         self.isa.has_x86_pmulhrsw_lowering()
     }
@@ -3323,6 +3319,214 @@ impl FuncEnvironment<'_> {
         let _ = (builder, num_pages, mem_index);
     }
 
+    pub fn ceil_f32(&mut self, builder: &mut FunctionBuilder, value: ir::Value) -> ir::Value {
+        // If the ISA has rounding instructions, let Cranelift use them. But if
+        // not, lower to a libcall here, rather than having Cranelift do it. We
+        // can pass our libcall the vmctx pointer, which we use for stack
+        // overflow checking.
+        if self.isa.has_round() {
+            builder.ins().ceil(value)
+        } else {
+            let ceil = self.builtin_functions.ceil_f32(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(ceil, &[vmctx, value]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn ceil_f64(&mut self, builder: &mut FunctionBuilder, value: ir::Value) -> ir::Value {
+        // See the comments in `ceil_f32` about libcalls.
+        if self.isa.has_round() {
+            builder.ins().ceil(value)
+        } else {
+            let ceil = self.builtin_functions.ceil_f64(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(ceil, &[vmctx, value]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn floor_f32(&mut self, builder: &mut FunctionBuilder, value: ir::Value) -> ir::Value {
+        // See the comments in `ceil_f32` about libcalls.
+        if self.isa.has_round() {
+            builder.ins().floor(value)
+        } else {
+            let floor = self.builtin_functions.floor_f32(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(floor, &[vmctx, value]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn floor_f64(&mut self, builder: &mut FunctionBuilder, value: ir::Value) -> ir::Value {
+        // See the comments in `ceil_f32` about libcalls.
+        if self.isa.has_round() {
+            builder.ins().floor(value)
+        } else {
+            let floor = self.builtin_functions.floor_f64(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(floor, &[vmctx, value]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn trunc_f32(&mut self, builder: &mut FunctionBuilder, value: ir::Value) -> ir::Value {
+        // See the comments in `ceil_f32` about libcalls.
+        if self.isa.has_round() {
+            builder.ins().trunc(value)
+        } else {
+            let trunc = self.builtin_functions.trunc_f32(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(trunc, &[vmctx, value]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn trunc_f64(&mut self, builder: &mut FunctionBuilder, value: ir::Value) -> ir::Value {
+        // See the comments in `ceil_f32` about libcalls.
+        if self.isa.has_round() {
+            builder.ins().trunc(value)
+        } else {
+            let trunc = self.builtin_functions.trunc_f64(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(trunc, &[vmctx, value]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn nearest_f32(&mut self, builder: &mut FunctionBuilder, value: ir::Value) -> ir::Value {
+        // See the comments in `ceil_f32` about libcalls.
+        if self.isa.has_round() {
+            builder.ins().nearest(value)
+        } else {
+            let nearest = self.builtin_functions.nearest_f32(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(nearest, &[vmctx, value]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn nearest_f64(&mut self, builder: &mut FunctionBuilder, value: ir::Value) -> ir::Value {
+        // See the comments in `ceil_f32` about libcalls.
+        if self.isa.has_round() {
+            builder.ins().nearest(value)
+        } else {
+            let nearest = self.builtin_functions.nearest_f64(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(nearest, &[vmctx, value]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn swizzle(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        a: ir::Value,
+        b: ir::Value,
+    ) -> ir::Value {
+        // On x86, swizzle would typically be compiled to `pshufb`, except
+        // that that's not available on CPUs that lack SSSE3. In that case,
+        // fall back to a builtin function.
+        if !self.is_x86() || self.isa.has_x86_pshufb_lowering() {
+            builder.ins().swizzle(a, b)
+        } else {
+            let swizzle = self.builtin_functions.i8x16_swizzle(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(swizzle, &[vmctx, a, b]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn relaxed_swizzle(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        a: ir::Value,
+        b: ir::Value,
+    ) -> ir::Value {
+        // As above, fall back to a builtin if we lack SSSE3.
+        if !self.is_x86() || self.isa.has_x86_pshufb_lowering() {
+            if !self.is_x86() || self.relaxed_simd_deterministic() {
+                builder.ins().swizzle(a, b)
+            } else {
+                builder.ins().x86_pshufb(a, b)
+            }
+        } else {
+            let swizzle = self.builtin_functions.i8x16_swizzle(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(swizzle, &[vmctx, a, b]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn i8x16_shuffle(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        a: ir::Value,
+        b: ir::Value,
+        lanes: &[u8; 16],
+    ) -> ir::Value {
+        // As with swizzle, i8x16.shuffle would also commonly be implemented
+        // with pshufb, so if we lack SSSE3, fall back to a builtin.
+        if !self.is_x86() || self.isa.has_x86_pshufb_lowering() {
+            let lanes = ConstantData::from(&lanes[..]);
+            let mask = builder.func.dfg.immediates.push(lanes);
+            builder.ins().shuffle(a, b, mask)
+        } else {
+            let lanes = builder
+                .func
+                .dfg
+                .constants
+                .insert(ConstantData::from(&lanes[..]));
+            let lanes = builder.ins().vconst(I8X16, lanes);
+            let i8x16_shuffle = self.builtin_functions.i8x16_shuffle(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(i8x16_shuffle, &[vmctx, a, b, lanes]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        }
+    }
+
+    pub fn fma_f32x4(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        a: ir::Value,
+        b: ir::Value,
+        c: ir::Value,
+    ) -> ir::Value {
+        if self.has_native_fma() {
+            builder.ins().fma(a, b, c)
+        } else if self.relaxed_simd_deterministic() {
+            // Deterministic semantics are "fused multiply and add".
+            let fma = self.builtin_functions.fma_f32x4(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(fma, &[vmctx, a, b, c]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        } else {
+            let mul = builder.ins().fmul(a, b);
+            builder.ins().fadd(mul, c)
+        }
+    }
+
+    pub fn fma_f64x2(
+        &mut self,
+        builder: &mut FunctionBuilder,
+        a: ir::Value,
+        b: ir::Value,
+        c: ir::Value,
+    ) -> ir::Value {
+        if self.has_native_fma() {
+            builder.ins().fma(a, b, c)
+        } else if self.relaxed_simd_deterministic() {
+            // Deterministic semantics are "fused multiply and add".
+            let fma = self.builtin_functions.fma_f64x2(builder.func);
+            let vmctx = self.vmctx_val(&mut builder.cursor());
+            let call = builder.ins().call(fma, &[vmctx, a, b, c]);
+            *builder.func.dfg.inst_results(call).first().unwrap()
+        } else {
+            let mul = builder.ins().fmul(a, b);
+            builder.ins().fadd(mul, c)
+        }
+    }
+
     pub fn isa(&self) -> &dyn TargetIsa {
         &*self.isa
     }

crates/cranelift/src/lib.rs

@@ -307,8 +307,9 @@ fn mach_reloc_to_reloc(
             }
         }
         FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {
-            let libcall = libcall_cranelift_to_wasmtime(libcall);
-            RelocationTarget::HostLibcall(libcall)
+            // We should have avoided any code that needs this style of libcalls
+            // in the Wasm-to-Cranelift translator.
+            panic!("unexpected libcall {libcall:?}");
         }
         _ => panic!("unrecognized external name"),
     };
@@ -320,24 +321,6 @@ fn mach_reloc_to_reloc(
     }
 }
 
-fn libcall_cranelift_to_wasmtime(call: ir::LibCall) -> wasmtime_environ::obj::LibCall {
-    use wasmtime_environ::obj::LibCall as LC;
-    match call {
-        ir::LibCall::FloorF32 => LC::FloorF32,
-        ir::LibCall::FloorF64 => LC::FloorF64,
-        ir::LibCall::NearestF32 => LC::NearestF32,
-        ir::LibCall::NearestF64 => LC::NearestF64,
-        ir::LibCall::CeilF32 => LC::CeilF32,
-        ir::LibCall::CeilF64 => LC::CeilF64,
-        ir::LibCall::TruncF32 => LC::TruncF32,
-        ir::LibCall::TruncF64 => LC::TruncF64,
-        ir::LibCall::FmaF32 => LC::FmaF32,
-        ir::LibCall::FmaF64 => LC::FmaF64,
-        ir::LibCall::X86Pshufb => LC::X86Pshufb,
-        _ => panic!("cranelift emitted a libcall wasmtime does not support: {call:?}"),
-    }
-}
-
 /// Helper structure for creating a `Signature` for all builtins.
 struct BuiltinFunctionSignatures {
     pointer_type: ir::Type,
@@ -373,10 +356,30 @@ impl BuiltinFunctionSignatures {
         AbiParam::new(ir::types::I64)
     }
 
+    fn f32(&self) -> AbiParam {
+        AbiParam::new(ir::types::F32)
+    }
+
+    fn f64(&self) -> AbiParam {
+        AbiParam::new(ir::types::F64)
+    }
+
     fn u8(&self) -> AbiParam {
         AbiParam::new(ir::types::I8)
     }
 
+    fn i8x16(&self) -> AbiParam {
+        AbiParam::new(ir::types::I8X16)
+    }
+
+    fn f32x4(&self) -> AbiParam {
+        AbiParam::new(ir::types::F32X4)
+    }
+
+    fn f64x2(&self) -> AbiParam {
+        AbiParam::new(ir::types::F64X2)
+    }
+
     fn bool(&self) -> AbiParam {
         AbiParam::new(ir::types::I8)
     }