Don't compute FnAbi for LLVM intrinsics

compiler/rustc_const_eval/src/interpret/call.rs

@@ -599,7 +599,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     pub(super) fn init_fn_call(
         &mut self,
         fn_val: FnVal<'tcx, M::ExtraFnVal>,
-        (caller_abi, caller_fn_abi): (ExternAbi, &FnAbi<'tcx, Ty<'tcx>>),
+        (caller_abi, caller_fn_abi): (ExternAbi, Option<&FnAbi<'tcx, Ty<'tcx>>>),
         args: &[FnArg<'tcx, M::Provenance>],
         with_caller_location: bool,
         destination: &PlaceTy<'tcx, M::Provenance>,
@@ -613,6 +613,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         let instance = match fn_val {
             FnVal::Instance(instance) => instance,
             FnVal::Other(extra) => {
+                let caller_fn_abi =
+                    caller_fn_abi.expect("FnAbi should have been computed for this call");
                 return M::call_extra_fn(
                     self,
                     extra,
@@ -677,6 +679,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             | ty::InstanceKind::Item(_) => {
                 // We need MIR for this fn.
                 // Note that this can be an intrinsic, if we are executing its fallback body.
+                let caller_fn_abi =
+                    caller_fn_abi.expect("FnAbi should have been computed for this call");
                 let Some((body, instance)) = M::find_mir_or_eval_fn(
                     self,
                     instance,
@@ -728,6 +732,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             // `InstanceKind::Virtual` does not have callable MIR. Calls to `Virtual` instances must be
             // codegen'd / interpreted as virtual calls through the vtable.
             ty::InstanceKind::Virtual(def_id, idx) => {
+                let caller_fn_abi =
+                    caller_fn_abi.expect("FnAbi should have been computed for this call");
                 let mut args = args.to_vec();
                 // We have to implement all "dyn-compatible receivers". So we have to go search for a
                 // pointer or `dyn Trait` type, but it could be wrapped in newtypes. So recursively
@@ -806,7 +812,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 // recurse with concrete function
                 self.init_fn_call(
                     FnVal::Instance(fn_inst),
-                    (caller_abi, &caller_fn_abi),
+                    (caller_abi, Some(&caller_fn_abi)),
                     &args,
                     with_caller_location,
                     destination,
@@ -849,7 +855,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     pub(super) fn init_fn_tail_call(
         &mut self,
         fn_val: FnVal<'tcx, M::ExtraFnVal>,
-        (caller_abi, caller_fn_abi): (ExternAbi, &FnAbi<'tcx, Ty<'tcx>>),
+        (caller_abi, caller_fn_abi): (ExternAbi, Option<&FnAbi<'tcx, Ty<'tcx>>>),
         args: &[FnArg<'tcx, M::Provenance>],
         with_caller_location: bool,
     ) -> InterpResult<'tcx> {
@@ -946,7 +952,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
         self.init_fn_call(
             FnVal::Instance(instance),
-            (ExternAbi::Rust, fn_abi),
+            (ExternAbi::Rust, Some(fn_abi)),
             &[FnArg::Copy(arg.into())],
             false,
             &ret.into(),

compiler/rustc_const_eval/src/interpret/step.rs

@@ -25,7 +25,8 @@ struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> {
     callee: FnVal<'tcx, M::ExtraFnVal>,
     args: Vec<FnArg<'tcx, M::Provenance>>,
     fn_sig: ty::FnSig<'tcx>,
-    fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
+    /// None if LLVM intrinsic
+    fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>,
     /// True if the function is marked as `#[track_caller]` ([`ty::InstanceKind::requires_caller_location`])
     with_caller_location: bool,
 }
@@ -480,13 +481,19 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             ty::FnPtr(..) => {
                 let fn_ptr = self.read_pointer(&func)?;
                 let fn_val = self.get_ptr_fn(fn_ptr)?;
-                (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false)
+                (fn_val, Some(self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?), false)
             }
             ty::FnDef(def_id, args) => {
                 let instance = self.resolve(def_id, args.no_bound_vars().unwrap())?;
+                // Don't compute FnAbi for LLVM intrinsics. Trying to do that would panic.
+                // Rust intrinsics however *do* need a FnAbi as we may invoke the
+                // fallback body like a regular function.
+                let has_fn_abi = !matches!(instance.def, ty::InstanceKind::LlvmIntrinsic(_));
                 (
                     FnVal::Instance(instance),
-                    self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)?,
+                    has_fn_abi
+                        .then(|| self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args))
+                        .transpose()?,
                     instance.def.requires_caller_location(*self.tcx),
                 )
             }
@@ -564,8 +571,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     with_caller_location,
                     &dest_place,
                     target,
-                    if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable },
+                    if fn_abi.map_or(false, |fn_abi| fn_abi.can_unwind) {
+                        unwind
+                    } else {
+                        mir::UnwindAction::Unreachable
+                    },
                 )?;
+
                 // Sanity-check that `eval_fn_call` either pushed a new frame or
                 // did a jump to another block. We disable the sanity check for functions that
                 // can't return, since Miri sometimes does have to keep the location the same

compiler/rustc_monomorphize/src/mono_checks/abi_check.rs

@@ -222,6 +222,10 @@ fn check_call_site_abi<'tcx>(
                 args.no_bound_vars().unwrap(),
                 DUMMY_SP,
             );
+            if let InstanceKind::LlvmIntrinsic(..) = instance.def {
+                // LLVM intrinsics don't have an ABI, so there is nothing to check.
+                return;
+            }
             tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty())))
         }
         _ => {

compiler/rustc_target/src/callconv/mod.rs

@@ -426,20 +426,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         PassMode::Indirect { attrs, meta_attrs, on_stack: false }
     }
 
-    /// Pass this argument directly instead. Should NOT be used!
-    /// Only exists because of past ABI mistakes that will take time to fix
-    /// (see <https://github.com/rust-lang/rust/issues/115666>).
-    #[track_caller]
-    pub fn make_direct_deprecated(&mut self) {
-        match self.mode {
-            PassMode::Indirect { .. } => {
-                self.mode = PassMode::Direct(ArgAttributes::new());
-            }
-            PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => {} // already direct
-            _ => panic!("Tried to make {:?} direct", self.mode),
-        }
-    }
-
     /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
     /// This is valid for both sized and unsized arguments.
     #[track_caller]

compiler/rustc_target/src/spec/abi_map.rs

@@ -97,6 +97,8 @@ impl AbiMap {
             // infallible lowerings
             (ExternAbi::C { .. }, _) => CanonAbi::C,
             (ExternAbi::Rust | ExternAbi::RustCall, _) => CanonAbi::Rust,
+
+            // Dummy mapping to prevent reporting an error in the frontend
             (ExternAbi::Unadjusted, _) => CanonAbi::C,
 
             (ExternAbi::RustCold, _) if self.os == OsKind::Windows => CanonAbi::Rust,

compiler/rustc_ty_utils/src/abi.rs

@@ -433,7 +433,6 @@ fn fn_abi_sanity_check<'tcx>(
 
     fn fn_arg_sanity_check<'tcx>(
         cx: &LayoutCx<'tcx>,
-        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         spec_abi: ExternAbi,
         arg: &ArgAbi<'tcx, Ty<'tcx>>,
         is_ret: bool,
@@ -466,28 +465,17 @@ fn fn_abi_sanity_check<'tcx>(
             PassMode::Direct(attrs) => {
                 // Here the Rust type is used to determine the actual ABI, so we have to be very
                 // careful. Scalar/Vector is fine, since backends will generally use
-                // `layout.backend_repr` and ignore everything else. We should just reject
-                //`Aggregate` entirely here, but some targets need to be fixed first.
+                // `layout.backend_repr` and ignore everything else.
                 match arg.layout.backend_repr {
                     BackendRepr::Scalar(_)
                     | BackendRepr::SimdVector { .. }
                     | BackendRepr::SimdScalableVector { .. } => {}
                     BackendRepr::ScalarPair { .. } => {
                         panic!("`PassMode::Direct` used for ScalarPair type {}", arg.layout.ty)
                     }
-                    BackendRepr::Memory { sized } => {
-                        // For an unsized type we'd only pass the sized prefix, so there is no universe
-                        // in which we ever want to allow this.
-                        assert!(sized, "`PassMode::Direct` for unsized type in ABI: {:#?}", fn_abi);
-
-                        // This really shouldn't happen even for sized aggregates, since
-                        // `immediate_llvm_type` will use `layout.fields` to turn this Rust type into an
-                        // LLVM type. This means all sorts of Rust type details leak into the ABI.
-                        // The unadjusted ABI however uses Direct for all args. It is ill-specified,
-                        // but unfortunately we need it for calling certain LLVM intrinsics.
-                        assert!(
-                            matches!(spec_abi, ExternAbi::Unadjusted),
-                            "`PassMode::Direct` for aggregates only allowed for \"unadjusted\"\n\
+                    BackendRepr::Memory { .. } => {
+                        panic!(
+                            "`PassMode::Direct` for aggregates not allowed\n\
                              Problematic type: {:#?}",
                             arg.layout,
                         );
@@ -539,9 +527,9 @@ fn fn_abi_sanity_check<'tcx>(
     }
 
     for arg in fn_abi.args.iter() {
-        fn_arg_sanity_check(cx, fn_abi, spec_abi, arg, false);
+        fn_arg_sanity_check(cx, spec_abi, arg, false);
     }
-    fn_arg_sanity_check(cx, fn_abi, spec_abi, &fn_abi.ret, true);
+    fn_arg_sanity_check(cx, spec_abi, &fn_abi.ret, true);
 }
 
 #[tracing::instrument(
@@ -636,26 +624,13 @@ fn fn_abi_adjust_for_abi<'tcx>(
     fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>,
     abi: ExternAbi,
 ) {
-    if abi == ExternAbi::Unadjusted {
-        // The "unadjusted" ABI passes aggregates in "direct" mode. That's fragile but needed for
-        // some LLVM intrinsics.
-        fn unadjust<'tcx>(arg: &mut ArgAbi<'tcx, Ty<'tcx>>) {
-            // This still uses `PassMode::Pair` for ScalarPair types. That's unlikely to be intended,
-            // but who knows what breaks if we change this now.
-            if matches!(arg.layout.backend_repr, BackendRepr::Memory { .. }) {
-                assert!(
-                    arg.layout.backend_repr.is_sized(),
-                    "'unadjusted' ABI does not support unsized arguments"
-                );
-            }
-            arg.make_direct_deprecated();
-        }
+    assert_ne!(
+        abi,
+        ExternAbi::Unadjusted,
+        "fn_abi_of_instance should not be called on LLVM intrinsics"
+    );
 
-        unadjust(&mut fn_abi.ret);
-        for arg in fn_abi.args.iter_mut() {
-            unadjust(arg);
-        }
-    } else if abi.is_rustic_abi() {
+    if abi.is_rustic_abi() {
         fn_abi.adjust_for_rust_abi(cx);
     } else {
         fn_abi.adjust_for_foreign_abi(cx, abi);

library/compiler-builtins/compiler-builtins/src/float/conv.rs

@@ -228,16 +228,26 @@ intrinsics! {
         f64::from_bits(int_to_float::u64_to_f64_bits(i))
     }
 
-    #[cfg_attr(target_os = "uefi", unadjusted_on_win64)]
+    #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))]
     pub extern "C" fn __floatuntisf(i: u128) -> f32 {
         f32::from_bits(int_to_float::u128_to_f32_bits(i))
     }
 
-    #[cfg_attr(target_os = "uefi", unadjusted_on_win64)]
+    #[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
+    pub extern "C" fn __floatuntisf(lo: u64, hi: u64) -> f32 {
+        f32::from_bits(int_to_float::u128_to_f32_bits((u128::from(hi) << 64) | u128::from(lo)))
+    }
+
+    #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))]
     pub extern "C" fn __floatuntidf(i: u128) -> f64 {
         f64::from_bits(int_to_float::u128_to_f64_bits(i))
     }
 
+    #[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
+    pub extern "C" fn __floatuntidf(lo: u64, hi: u64) -> f64 {
+        f64::from_bits(int_to_float::u128_to_f64_bits((u128::from(hi) << 64) | u128::from(lo)))
+    }
+
     #[ppc_alias = __floatunsikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatunsitf(i: u32) -> f128 {
@@ -279,16 +289,26 @@ intrinsics! {
         int_to_float::signed(i, int_to_float::u64_to_f64_bits)
     }
 
-    #[cfg_attr(target_os = "uefi", unadjusted_on_win64)]
+    #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))]
     pub extern "C" fn __floattisf(i: i128) -> f32 {
         int_to_float::signed(i, int_to_float::u128_to_f32_bits)
     }
 
-    #[cfg_attr(target_os = "uefi", unadjusted_on_win64)]
+    #[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
+    pub extern "C" fn __floattisf(lo: u64, hi: u64) -> f32 {
+        int_to_float::signed((i128::from(hi) << 64) | i128::from(lo), int_to_float::u128_to_f32_bits)
+    }
+
+    #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))]
     pub extern "C" fn __floattidf(i: i128) -> f64 {
         int_to_float::signed(i, int_to_float::u128_to_f64_bits)
     }
 
+    #[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
+    pub extern "C" fn __floattidf(lo: u64, hi: u64) -> f64 {
+        int_to_float::signed((i128::from(hi) << 64) | i128::from(lo), int_to_float::u128_to_f64_bits)
+    }
+
     #[ppc_alias = __floatsikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatsitf(i: i32) -> f128 {

library/compiler-builtins/compiler-builtins/src/int/mul.rs

@@ -173,13 +173,29 @@ intrinsics! {
         mul
     }
 
-    #[unadjusted_on_win64]
+    #[cfg(not(all(
+        any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")),
+        target_pointer_width = "64",
+    )))]
     pub extern "C" fn __muloti4(a: i128, b: i128, oflow: &mut i32) -> i128 {
         let (mul, o) = i128_overflowing_mul(a, b);
         *oflow = o as i32;
         mul
     }
 
+    #[cfg(all(
+        any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")),
+        target_pointer_width = "64",
+    ))]
+    pub extern "C" fn f(a_lo: u64, a_hi: u64, b_lo: u64, b_hi: u64, oflow: &mut i32) -> i128 {
+        let (mul, o) = i128_overflowing_mul(
+            (i128::from(a_hi) << 64) | i128::from(a_lo),
+            (i128::from(b_hi) << 64) | i128::from(b_lo),
+        );
+        *oflow = o as i32;
+        mul
+    }
+
     pub extern "C" fn __rust_i128_mulo(a: i128, b: i128, oflow: &mut i32) -> i128 {
         let (mul, o) = i128_overflowing_mul(a, b);
         *oflow = o.into();

library/compiler-builtins/compiler-builtins/src/macros.rs

@@ -108,6 +108,25 @@ macro_rules! intrinsics {
 
         intrinsics!($($rest)*);
     );
+    // Support cfg:
+    (
+        #[cfg($e:meta)]
+        $(#[$($attrs:tt)*])*
+        pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
+            $($body:tt)*
+        }
+        $($rest:tt)*
+    ) => (
+        #[cfg($e)]
+        intrinsics! {
+            $(#[$($attrs)*])*
+            pub extern $abi fn $name($($argname: $ty),*) $(-> $ret)? {
+                $($body)*
+            }
+        }
+
+        intrinsics!($($rest)*);
+    );
 
     // Right now there's a bunch of architecture-optimized intrinsics in the
     // stock compiler-rt implementation. Not all of these have been ported over
@@ -182,36 +201,6 @@ macro_rules! intrinsics {
         intrinsics!($($rest)*);
     );
 
-    // Like aapcs above we recognize an attribute for the "unadjusted" abi on
-    // win64 for some methods.
-    (
-        #[unadjusted_on_win64]
-        $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
-            $($body:tt)*
-        }
-
-        $($rest:tt)*
-    ) => (
-        #[cfg(all(any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")), target_pointer_width = "64"))]
-        intrinsics! {
-            $(#[$($attr)*])*
-            pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? {
-                $($body)*
-            }
-        }
-
-        #[cfg(not(all(any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")), target_pointer_width = "64")))]
-        intrinsics! {
-            $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
-                $($body)*
-            }
-        }
-
-        intrinsics!($($rest)*);
-    );
-
     // `arm_aeabi_alias` would conflict with `f16_apple_{arg,ret}_abi` not handled here. Avoid macro ambiguity by combining in a
     // single `#[]`.
     (