Don't compute FnAbi for LLVM intrinsics

Author: bjorn3

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -4,9 +4,9 @@ mod simd;
 #[cfg(feature = "master")]
 use std::iter;
 
-use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp};
 #[cfg(feature = "master")]
-use gccjit::{FnAttribute, Type};
+use gccjit::Type;
+use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp};
 #[cfg(feature = "master")]
 use rustc_abi::ExternAbi;
 use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange};
@@ -24,16 +24,16 @@ use rustc_codegen_ssa::traits::{
 };
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::bug;
-use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
-use rustc_middle::ty::{self, Instance, Ty};
 #[cfg(feature = "master")]
-use rustc_session::config;
+use rustc_middle::ty::layout::FnAbiOf;
+use rustc_middle::ty::layout::LayoutOf;
+use rustc_middle::ty::{self, Instance, Ty};
 use rustc_span::{Span, Symbol, sym};
-#[cfg(feature = "master")]
-use rustc_target::callconv::ArgAttributes;
 use rustc_target::callconv::{ArgAbi, PassMode};
 
-use crate::abi::{FnAbiGcc, FnAbiGccExt, GccType};
+#[cfg(feature = "master")]
+use crate::abi::FnAbiGccExt;
+use crate::abi::GccType;
 use crate::builder::Builder;
 use crate::common::{SignType, TypeReflection};
 use crate::context::CodegenCx;
@@ -625,83 +625,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
                 *func
             } else {
                 self.linkage.set(FunctionType::Extern);
-                let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
-                assert!(!fn_abi.ret.is_indirect());
-                assert!(!fn_abi.c_variadic);
-
-                let return_type = match fn_abi.ret.mode {
-                    PassMode::Ignore => self.type_void(),
-                    PassMode::Direct(_) | PassMode::Pair(..) => {
-                        fn_abi.ret.layout.immediate_gcc_type(self)
-                    }
-                    PassMode::Cast { .. } | PassMode::Indirect { .. } => {
-                        unreachable!()
-                    }
-                };
-
-                #[cfg(feature = "master")]
-                let mut non_null_args = Vec::new();
-
-                #[cfg(feature = "master")]
-                let mut apply_attrs =
-                    |mut ty: Type<'gcc>, attrs: &ArgAttributes, arg_index: usize| {
-                        if self.sess().opts.optimize == config::OptLevel::No {
-                            return ty;
-                        }
-                        if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NoAlias) {
-                            ty = ty.make_restrict()
-                        }
-                        if attrs.regular.contains(rustc_target::callconv::ArgAttribute::NonNull) {
-                            non_null_args.push(arg_index as i32 + 1);
-                        }
-                        ty
-                    };
-                #[cfg(not(feature = "master"))]
-                let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes, _arg_index: usize| ty;
-
-                let mut argument_tys = Vec::with_capacity(fn_abi.args.len());
-                for arg in fn_abi.args.iter() {
-                    match arg.mode {
-                        PassMode::Ignore => {}
-                        PassMode::Pair(a, b) => {
-                            let arg_pos = argument_tys.len();
-                            argument_tys.push(apply_attrs(
-                                arg.layout.scalar_pair_element_gcc_type(self, 0),
-                                &a,
-                                arg_pos,
-                            ));
-                            argument_tys.push(apply_attrs(
-                                arg.layout.scalar_pair_element_gcc_type(self, 1),
-                                &b,
-                                arg_pos + 1,
-                            ));
-                        }
-                        PassMode::Direct(attrs) => argument_tys.push(apply_attrs(
-                            arg.layout.immediate_gcc_type(self),
-                            &attrs,
-                            argument_tys.len(),
-                        )),
-                        PassMode::Indirect { .. } | PassMode::Cast { .. } => {
-                            unreachable!()
-                        }
-                    }
-                }
-
-                #[cfg(feature = "master")]
-                let fn_attrs = if non_null_args.is_empty() {
-                    Vec::new()
-                } else {
-                    vec![FnAttribute::NonNull(non_null_args)]
-                };
-
-                let fn_ty = FnAbiGcc {
-                    return_type,
-                    arguments_type: argument_tys,
-                    is_c_variadic: false,
-                    on_stack_param_indices: FxHashSet::default(),
-                    #[cfg(feature = "master")]
-                    fn_attributes: fn_attrs,
-                };
 
                 let func = match sym {
                     "llvm.fma.f16" => {
@@ -714,13 +637,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
 
                 self.intrinsics.borrow_mut().insert(sym.to_string(), func);
 
-                self.on_stack_function_params
-                    .borrow_mut()
-                    .insert(func, fn_ty.on_stack_param_indices);
-                #[cfg(feature = "master")]
-                for fn_attr in fn_ty.fn_attributes {
-                    func.add_attribute(fn_attr);
-                }
+                self.on_stack_function_params.borrow_mut().insert(func, FxHashSet::default());
 
                 crate::attributes::from_fn_attrs(self, func, instance);
 

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -646,32 +646,34 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     ) -> Self::Value {
         let tcx = self.tcx();
 
-        // FIXME remove usage of fn_abi
-        let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
-        assert!(!fn_abi.ret.is_indirect());
-        assert!(!fn_abi.c_variadic);
-
-        let llreturn_ty = match &fn_abi.ret.mode {
-            PassMode::Ignore => self.type_void(),
-            PassMode::Direct(_) | PassMode::Pair(..) => fn_abi.ret.layout.immediate_llvm_type(self),
-            PassMode::Cast { .. } | PassMode::Indirect { .. } => {
-                unreachable!()
+        let fn_ty = instance.ty(tcx, self.typing_env());
+        let fn_sig = match *fn_ty.kind() {
+            ty::FnDef(def_id, args) => {
+                tcx.instantiate_bound_regions_with_erased(tcx.fn_sig(def_id).instantiate(tcx, args))
             }
+            _ => unreachable!(),
         };
+        assert!(!fn_sig.c_variadic);
 
-        let mut llargument_tys = Vec::with_capacity(fn_abi.args.len());
-        for arg in &fn_abi.args {
-            match &arg.mode {
-                PassMode::Ignore => {}
-                PassMode::Direct(_) => llargument_tys.push(arg.layout.immediate_llvm_type(self)),
-                PassMode::Pair(..) => {
-                    llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(self, 0, true));
-                    llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(self, 1, true));
-                }
-                PassMode::Indirect { .. } | PassMode::Cast { .. } => {
-                    unreachable!()
-                }
-            };
+        let ret_layout = self.layout_of(fn_sig.output());
+        let llreturn_ty = if ret_layout.is_zst() {
+            self.type_void()
+        } else {
+            ret_layout.immediate_llvm_type(self)
+        };
+
+        let mut llargument_tys = Vec::with_capacity(fn_sig.inputs().len());
+        for &arg in fn_sig.inputs() {
+            let arg_layout = self.layout_of(arg);
+            if arg_layout.is_zst() {
+                continue;
+            }
+            if let BackendRepr::ScalarPair(_, _) = arg_layout.backend_repr {
+                llargument_tys.push(arg_layout.scalar_pair_element_llvm_type(self, 0, true));
+                llargument_tys.push(arg_layout.scalar_pair_element_llvm_type(self, 1, true));
+                continue;
+            }
+            llargument_tys.push(arg_layout.immediate_llvm_type(self));
         }
 
         let fn_ty = self.type_func(&llargument_tys, llreturn_ty);

compiler/rustc_monomorphize/src/mono_checks/abi_check.rs

@@ -207,6 +207,13 @@ fn check_call_site_abi<'tcx>(
                 return;
             }
             let instance = ty::Instance::expect_resolve(tcx, typing_env, def_id, args, DUMMY_SP);
+
+            if let Some(name) = tcx.codegen_fn_attrs(instance.def_id()).symbol_name
+                && name.as_str().starts_with("llvm.")
+            {
+                return;
+            }
+
             tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty())))
         }
         _ => {

compiler/rustc_target/src/callconv/mod.rs

@@ -417,20 +417,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         PassMode::Indirect { attrs, meta_attrs, on_stack: false }
     }
 
-    /// Pass this argument directly instead. Should NOT be used!
-    /// Only exists because of past ABI mistakes that will take time to fix
-    /// (see <https://github.com/rust-lang/rust/issues/115666>).
-    #[track_caller]
-    pub fn make_direct_deprecated(&mut self) {
-        match self.mode {
-            PassMode::Indirect { .. } => {
-                self.mode = PassMode::Direct(ArgAttributes::new());
-            }
-            PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => {} // already direct
-            _ => panic!("Tried to make {:?} direct", self.mode),
-        }
-    }
-
     /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
     /// This is valid for both sized and unsized arguments.
     #[track_caller]

compiler/rustc_target/src/spec/abi_map.rs

@@ -84,6 +84,8 @@ impl AbiMap {
             // infallible lowerings
             (ExternAbi::C { .. }, _) => CanonAbi::C,
             (ExternAbi::Rust | ExternAbi::RustCall, _) => CanonAbi::Rust,
+
+            // Dummy mapping to prevent reporting an error in the frontend
             (ExternAbi::Unadjusted, _) => CanonAbi::C,
 
             (ExternAbi::RustCold, _) if self.os == OsKind::Windows => CanonAbi::Rust,

compiler/rustc_ty_utils/src/abi.rs

@@ -379,7 +379,6 @@ fn fn_abi_sanity_check<'tcx>(
 ) {
     fn fn_arg_sanity_check<'tcx>(
         cx: &LayoutCx<'tcx>,
-        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         spec_abi: ExternAbi,
         arg: &ArgAbi<'tcx, Ty<'tcx>>,
     ) {
@@ -409,28 +408,17 @@ fn fn_abi_sanity_check<'tcx>(
             PassMode::Direct(_) => {
                 // Here the Rust type is used to determine the actual ABI, so we have to be very
                 // careful. Scalar/Vector is fine, since backends will generally use
-                // `layout.backend_repr` and ignore everything else. We should just reject
-                //`Aggregate` entirely here, but some targets need to be fixed first.
+                // `layout.backend_repr` and ignore everything else.
                 match arg.layout.backend_repr {
                     BackendRepr::Scalar(_)
                     | BackendRepr::SimdVector { .. }
                     | BackendRepr::ScalableVector { .. } => {}
                     BackendRepr::ScalarPair(..) => {
                         panic!("`PassMode::Direct` used for ScalarPair type {}", arg.layout.ty)
                     }
-                    BackendRepr::Memory { sized } => {
-                        // For an unsized type we'd only pass the sized prefix, so there is no universe
-                        // in which we ever want to allow this.
-                        assert!(sized, "`PassMode::Direct` for unsized type in ABI: {:#?}", fn_abi);
-
-                        // This really shouldn't happen even for sized aggregates, since
-                        // `immediate_llvm_type` will use `layout.fields` to turn this Rust type into an
-                        // LLVM type. This means all sorts of Rust type details leak into the ABI.
-                        // The unadjusted ABI however uses Direct for all args. It is ill-specified,
-                        // but unfortunately we need it for calling certain LLVM intrinsics.
-                        assert!(
-                            matches!(spec_abi, ExternAbi::Unadjusted),
-                            "`PassMode::Direct` for aggregates only allowed for \"unadjusted\"\n\
+                    BackendRepr::Memory { .. } => {
+                        panic!(
+                            "`PassMode::Direct` for aggregates not allowed\n\
                              Problematic type: {:#?}",
                             arg.layout,
                         );
@@ -474,9 +462,9 @@ fn fn_abi_sanity_check<'tcx>(
     }
 
     for arg in fn_abi.args.iter() {
-        fn_arg_sanity_check(cx, fn_abi, spec_abi, arg);
+        fn_arg_sanity_check(cx, spec_abi, arg);
     }
-    fn_arg_sanity_check(cx, fn_abi, spec_abi, &fn_abi.ret);
+    fn_arg_sanity_check(cx, spec_abi, &fn_abi.ret);
 }
 
 #[tracing::instrument(level = "debug", skip(cx, instance))]
@@ -603,27 +591,11 @@ fn fn_abi_adjust_for_abi<'tcx>(
     abi: ExternAbi,
     fn_def_id: Option<DefId>,
 ) {
-    if abi == ExternAbi::Unadjusted {
-        // The "unadjusted" ABI passes aggregates in "direct" mode. That's fragile but needed for
-        // some LLVM intrinsics.
-        fn unadjust<'tcx>(arg: &mut ArgAbi<'tcx, Ty<'tcx>>) {
-            // This still uses `PassMode::Pair` for ScalarPair types. That's unlikely to be intended,
-            // but who knows what breaks if we change this now.
-            if matches!(arg.layout.backend_repr, BackendRepr::Memory { .. }) {
-                assert!(
-                    arg.layout.backend_repr.is_sized(),
-                    "'unadjusted' ABI does not support unsized arguments"
-                );
-            }
-            arg.make_direct_deprecated();
-        }
-
-        unadjust(&mut fn_abi.ret);
-        for arg in fn_abi.args.iter_mut() {
-            unadjust(arg);
-        }
-        return;
-    }
+    assert_ne!(
+        abi,
+        ExternAbi::Unadjusted,
+        "fn_abi_of_instance should not be called on LLVM intrinsics"
+    );
 
     let tcx = cx.tcx();