Move loop_hint_attrs from Terminator into TerminatorKind::Goto

compiler/rustc_borrowck/src/lib.rs

@@ -978,7 +978,7 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a,
                 }
             }
 
-            TerminatorKind::Goto { target: _ }
+            TerminatorKind::Goto { target: _, loop_hint_attrs: _ }
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Unreachable
             | TerminatorKind::UnwindResume

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -195,7 +195,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
                     }
                 }
             }
-            TerminatorKind::Goto { target: _ }
+            TerminatorKind::Goto { target: _, loop_hint_attrs: _ }
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Unreachable
             | TerminatorKind::FalseEdge { real_target: _, imaginary_target: _ }

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -2082,7 +2082,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     fn check_iscleanup(&mut self, block_data: &BasicBlockData<'tcx>) {
         let is_cleanup = block_data.is_cleanup;
         match block_data.terminator().kind {
-            TerminatorKind::Goto { target } => {
+            TerminatorKind::Goto { target, .. } => {
                 self.assert_iscleanup(block_data, target, is_cleanup)
             }
             TerminatorKind::SwitchInt { ref targets, .. } => {

compiler/rustc_codegen_cranelift/src/base.rs

@@ -343,7 +343,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             crate::PrintOnPanic(|| format!("terminator {:?}", bb_data.terminator().kind));
 
         match &bb_data.terminator().kind {
-            TerminatorKind::Goto { target } => {
+            TerminatorKind::Goto { target, loop_hint_attrs: _ } => {
                 if let TerminatorKind::Return = fx.mir[*target].terminator().kind {
                     let mut can_immediately_return = true;
                     for stmt in &fx.mir[*target].statements {

compiler/rustc_codegen_llvm/src/builder.rs

@@ -8,7 +8,7 @@ pub(crate) mod gpu_offload;
 
 use libc::{c_char, c_uint};
 use rustc_abi::{self as abi, Align, CanonAbi, Size, WrappingRange};
-use rustc_attr_ir::{AttributeKind, UnrollAttr};
+use rustc_attr_ir::UnrollAttr;
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
@@ -340,16 +340,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
-    fn br_with_attrs(&mut self, dest: &'ll BasicBlock, loop_hint_attrs: &[AttributeKind]) {
+    fn br_with_attrs(&mut self, dest: &'ll BasicBlock, loop_hint_attrs: &[UnrollAttr]) {
         unsafe {
             let val = llvm::LLVMBuildBr(self.llbuilder, dest);
 
             let mut nodes = Vec::new();
 
-            for loop_hint_attr in loop_hint_attrs {
-                let AttributeKind::Unroll(unroll) = loop_hint_attr else {
-                    continue;
-                };
+            for unroll in loop_hint_attrs {
                 // UnrollAttr::Count needs a second operand, the provided count, but the other
                 // unroll hints do not.
                 let md_node = if let UnrollAttr::Count(count) = unroll {

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -7,7 +7,7 @@ use rustc_abi::{
 };
 use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
-use rustc_attr_ir::AttributeKind;
+use rustc_attr_ir::UnrollAttr;
 use rustc_attr_ir::lang_items::LangItem;
 use rustc_data_structures::packed::Pu128;
 use rustc_lint_defs::builtin::TAIL_CALL_TRACK_CALLER;
@@ -140,7 +140,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         bx: &mut Bx,
         target: mir::BasicBlock,
         mergeable_succ: bool,
-        loop_hint_attrs: &[AttributeKind],
+        loop_hint_attrs: &[UnrollAttr],
     ) -> MergingSucc {
         let (needs_landing_pad, is_cleanupret) = self.llbb_characteristics(fx, target);
         if mergeable_succ && !needs_landing_pad && !is_cleanupret {
@@ -1676,8 +1676,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 MergingSucc::False
             }
 
-            mir::TerminatorKind::Goto { target } => {
-                helper.funclet_br(self, bx, target, mergeable_succ(), &terminator.loop_hint_attrs)
+            mir::TerminatorKind::Goto { target, ref loop_hint_attrs } => {
+                helper.funclet_br(self, bx, target, mergeable_succ(), loop_hint_attrs)
             }
 
             mir::TerminatorKind::SwitchInt { ref discr, ref targets } => {

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -437,7 +437,7 @@ fn optimize_use_clone<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 ))),
             ));
 
-            bb.terminator_mut().kind = mir::TerminatorKind::Goto { target: destination_block };
+            bb.terminator_mut().kind = mir::TerminatorKind::goto(destination_block);
         }
     }
 

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -3,7 +3,7 @@ use std::ops::Deref;
 
 use rustc_abi::{Align, Scalar, Size, WrappingRange};
 use rustc_ast::expand::typetree::{FncTree, TypeTree};
-use rustc_attr_ir::AttributeKind;
+use rustc_attr_ir::UnrollAttr;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::mir;
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
@@ -94,7 +94,7 @@ pub trait BuilderMethods<'a, 'tcx>:
     fn ret_void(&mut self);
     fn ret(&mut self, v: Self::Value);
     fn br(&mut self, dest: Self::BasicBlock);
-    fn br_with_attrs(&mut self, dest: Self::BasicBlock, _loop_hint_attrs: &[AttributeKind]) {
+    fn br_with_attrs(&mut self, dest: Self::BasicBlock, _loop_hint_attrs: &[UnrollAttr]) {
         self.br(dest)
     }
     fn cond_br(

compiler/rustc_const_eval/src/interpret/step.rs

@@ -532,7 +532,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 self.return_from_current_stack_frame(/* unwinding */ false)?
             }
 
-            Goto { target } => self.go_to_block(target),
+            Goto { target, loop_hint_attrs: _ } => self.go_to_block(target),
 
             SwitchInt { ref discr, ref targets } => {
                 let discr = self.read_immediate(&self.eval_operand(discr, None)?)?;

compiler/rustc_middle/src/mir/mod.rs

@@ -1732,11 +1732,11 @@ mod size_asserts {
 
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 144);
+    static_assert_size!(BasicBlockData<'_>, 136);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 64);
     static_assert_size!(Statement<'_>, 40);
-    static_assert_size!(Terminator<'_>, 104);
+    static_assert_size!(Terminator<'_>, 96);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -5,6 +5,7 @@
 
 use rustc_abi::{FieldIdx, VariantIdx};
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece, Mutability};
+use rustc_attr_ir::UnrollAttr;
 use rustc_data_structures::packed::Pu128;
 use rustc_hir::CoroutineKind;
 use rustc_hir::def_id::DefId;
@@ -14,6 +15,7 @@ use rustc_span::def_id::LocalDefId;
 use rustc_span::{Span, Spanned, Symbol};
 use rustc_target::asm::InlineAsmRegOrRegClass;
 use smallvec::SmallVec;
+use thin_vec::ThinVec;
 
 use super::{BasicBlock, Const, Local, UserTypeProjection};
 use crate::mir::coverage::CoverageKind;
@@ -683,7 +685,7 @@ pub enum InlineAsmMacro {
 #[derive(Clone, TyEncodable, TyDecodable, StableHash, PartialEq, TypeFoldable, TypeVisitable)]
 pub enum TerminatorKind<'tcx> {
     /// Block has one successor; we continue execution there.
-    Goto { target: BasicBlock },
+    Goto { target: BasicBlock, loop_hint_attrs: ThinVec<UnrollAttr> },
 
     /// Switches based on the computed value.
     ///

compiler/rustc_middle/src/mir/terminator.rs

@@ -3,7 +3,6 @@
 use std::slice;
 
 use rustc_ast::InlineAsmOptions;
-use rustc_attr_ir::AttributeKind;
 use rustc_attr_ir::lang_items::LangItem;
 use rustc_data_structures::packed::Pu128;
 use rustc_macros::{StableHash, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
@@ -419,7 +418,6 @@ impl<O: fmt::Debug> fmt::Display for AssertKind<O> {
 pub struct Terminator<'tcx> {
     pub source_info: SourceInfo,
     pub kind: TerminatorKind<'tcx>,
-    pub loop_hint_attrs: ThinVec<AttributeKind>,
 }
 
 impl<'tcx> Terminator<'tcx> {
@@ -479,6 +477,11 @@ impl<'tcx> TerminatorKind<'tcx> {
     pub fn if_(cond: Operand<'tcx>, t: BasicBlock, f: BasicBlock) -> TerminatorKind<'tcx> {
         TerminatorKind::SwitchInt { discr: cond, targets: SwitchTargets::static_if(0, f, t) }
     }
+
+    #[inline]
+    pub fn goto(target: BasicBlock) -> TerminatorKind<'tcx> {
+        TerminatorKind::Goto { target, loop_hint_attrs: ThinVec::new() }
+    }
 }
 
 pub use helper::*;
@@ -527,7 +530,7 @@ mod helper {
                     mk_successors(slice::from_ref(t), Some(u), None)
                 }
                 // single successor
-                Goto { target: ref t }
+                Goto { target: ref t, .. }
                 | Call { target: None, unwind: UnwindAction::Cleanup(ref t), .. }
                 | Call { target: Some(ref t), unwind: _, .. }
                 | Yield { resume: ref t, drop: None, .. }
@@ -590,7 +593,7 @@ mod helper {
                         f(u)
                     }
                 }
-                Goto { target } => {
+                Goto { target, .. } => {
                     f(target);
                 }
                 UnwindResume
@@ -675,7 +678,7 @@ impl<'tcx> TerminatorKind<'tcx> {
     #[inline]
     pub fn as_goto(&self) -> Option<BasicBlock> {
         match self {
-            TerminatorKind::Goto { target } => Some(*target),
+            TerminatorKind::Goto { target, loop_hint_attrs: _ } => Some(*target),
             _ => None,
         }
     }
@@ -745,7 +748,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             | CoroutineDrop
             | Unreachable => TerminatorEdges::None,
 
-            Goto { target } => TerminatorEdges::Single(target),
+            Goto { target, loop_hint_attrs: _ } => TerminatorEdges::Single(target),
 
             // FIXME: Maybe we need also TerminatorEdges::Trio for async drop
             // (target + unwind + dropline)

compiler/rustc_middle/src/mir/visit.rs

@@ -518,7 +518,7 @@ macro_rules! make_mir_visitor {
                 terminator: &$($mutability)? Terminator<'tcx>,
                 location: Location
             ) {
-                let Terminator { source_info, kind, loop_hint_attrs: _ } = terminator;
+                let Terminator { source_info, kind } = terminator;
 
                 self.visit_source_info(source_info);
                 match kind {

compiler/rustc_middle/src/thir.rs

@@ -15,7 +15,7 @@ use std::sync::Arc;
 
 use rustc_abi::{FieldIdx, Integer, Size, VariantIdx};
 use rustc_ast::{AsmMacro, InlineAsmOptions, InlineAsmTemplatePiece, Mutability};
-use rustc_attr_ir::AttributeKind;
+use rustc_attr_ir::UnrollAttr;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_hir as hir;
@@ -62,7 +62,7 @@ macro_rules! thir_with_elements {
         #[derive(Debug, StableHash, Clone)]
         pub struct Thir<'tcx> {
             pub body_type: BodyTy<'tcx>,
-            pub loop_hint_attrs: FxIndexMap<ExprId, ThinVec<AttributeKind>>,
+            pub loop_hint_attrs: FxIndexMap<ExprId, ThinVec<UnrollAttr>>,
             $(
                 pub $name: IndexVec<$id, $value>,
             )*

compiler/rustc_middle/src/ty/structural_impls.rs

@@ -218,6 +218,7 @@ TrivialTypeTraversalImpls! {
     rustc_ast::InlineAsmOptions,
     rustc_ast::InlineAsmTemplatePiece,
     rustc_attr_ir::AttributeKind,
+    rustc_attr_ir::UnrollAttr,
     rustc_hir::CoroutineKind,
     rustc_hir::HirId,
     rustc_hir::MatchSource,

compiler/rustc_mir_build/src/builder/cfg.rs

@@ -1,5 +1,6 @@
 //! Routines for manipulating the control-flow graph.
 
+use rustc_attr_ir::UnrollAttr;
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
@@ -120,8 +121,7 @@ impl<'tcx> CFG<'tcx> {
             block,
             self.block_data(block)
         );
-        self.block_data_mut(block).terminator =
-            Some(Terminator { source_info, kind, loop_hint_attrs: ThinVec::new() });
+        self.block_data_mut(block).terminator = Some(Terminator { source_info, kind });
         self.block_data_mut(block).terminator.as_mut().unwrap()
     }
 
@@ -132,6 +132,17 @@ impl<'tcx> CFG<'tcx> {
         source_info: SourceInfo,
         target: BasicBlock,
     ) -> &mut Terminator<'tcx> {
-        self.terminate(origin, source_info, TerminatorKind::Goto { target })
+        self.goto_with_hints(origin, source_info, target, ThinVec::new())
+    }
+
+    #[inline(always)]
+    pub(crate) fn goto_with_hints(
+        &mut self,
+        origin: BasicBlock,
+        source_info: SourceInfo,
+        target: BasicBlock,
+        loop_hint_attrs: ThinVec<UnrollAttr>,
+    ) -> &mut Terminator<'tcx> {
+        self.terminate(origin, source_info, TerminatorKind::Goto { target, loop_hint_attrs })
     }
 }

compiler/rustc_mir_build/src/builder/custom/parse.rs

@@ -1,4 +1,3 @@
-use rustc_data_structures::thin_vec::ThinVec;
 use rustc_index::IndexSlice;
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
@@ -319,7 +318,6 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
         data.terminator = Some(Terminator {
             source_info: SourceInfo { span, scope: self.source_scope },
             kind: terminator,
-            loop_hint_attrs: ThinVec::new(),
         });
 
         Ok(data)

compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs

@@ -47,7 +47,7 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
                 Ok(TerminatorKind::Return)
             },
             @call(mir_goto, args) => {
-                Ok(TerminatorKind::Goto { target: self.parse_block(args[0])? } )
+                Ok(TerminatorKind::goto(self.parse_block(args[0])?))
             },
             @call(mir_unreachable, _args) => {
                 Ok(TerminatorKind::Unreachable)

compiler/rustc_mir_build/src/builder/expr/into.rs

@@ -4,6 +4,7 @@ use rustc_abi::FieldIdx;
 use rustc_ast::{AsmMacro, InlineAsmOptions};
 use rustc_attr_ir::lang_items::LangItem;
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::thin_vec::ThinVec;
 use rustc_hir as hir;
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
@@ -237,10 +238,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // Execute the body, branching back to the test.
                     let body_block_end = this.expr_into_dest(tmp, body_block, body).into_block();
 
-                    let goto = this.cfg.goto(body_block_end, source_info, loop_block);
-                    if let Some(attrs) = this.thir.loop_hint_attrs.get(&expr_id) {
-                        goto.loop_hint_attrs = attrs.clone();
-                    }
+                    let loop_hint_attrs = this
+                        .thir
+                        .loop_hint_attrs
+                        .get(&expr_id)
+                        .map(|attrs| attrs.clone())
+                        .unwrap_or_else(|| ThinVec::new());
+                    this.cfg.goto_with_hints(
+                        body_block_end,
+                        source_info,
+                        loop_block,
+                        loop_hint_attrs,
+                    );
 
                     // Loops are only exited by `break` expressions.
                     None
@@ -772,7 +781,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             this.cfg.terminate(
                                 target,
                                 source_info,
-                                TerminatorKind::Goto { target: destination_block },
+                                TerminatorKind::goto(destination_block),
                             );
 
                             mir::InlineAsmOperand::Label { target_index }