Implement borrowck for &pin mut|const $place

compiler/rustc_borrowck/src/borrow_set.rs

@@ -124,6 +124,8 @@ impl<'tcx> fmt::Display for BorrowData<'tcx> {
             mir::BorrowKind::Mut {
                 kind: mir::MutBorrowKind::Default | mir::MutBorrowKind::TwoPhaseBorrow,
             } => "mut ",
+            mir::BorrowKind::Pinned(mir::Mutability::Not, _) => "pin const ",
+            mir::BorrowKind::Pinned(mir::Mutability::Mut, _) => "pin mut ",
         };
         write!(w, "&{:?} {}{:?}", self.region, kind, self.borrowed_place)
     }

compiler/rustc_borrowck/src/borrowck_errors.rs

@@ -484,6 +484,38 @@ impl<'infcx, 'tcx> crate::MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     pub(crate) fn temporary_value_borrowed_for_too_long(&self, span: Span) -> Diag<'infcx> {
         struct_span_code_err!(self.dcx(), span, E0716, "temporary value dropped while borrowed")
     }
+
+    pub(crate) fn cannot_move_out_while_pinned(
+        &self,
+        span: Span,
+        pin_span: Span,
+        place: &str,
+        pin_place: &str,
+        value_place: &str,
+    ) -> Diag<'infcx> {
+        self.dcx().create_err(crate::session_diagnostics::MovePin {
+            place,
+            pin_place,
+            value_place,
+            span,
+            pin_span,
+        })
+    }
+
+    pub(crate) fn cannot_mutably_borrow_pinned(
+        &self,
+        span: Span,
+        pin_span: Span,
+        place: &str,
+        pin_place: &str,
+    ) -> Diag<'infcx> {
+        self.dcx().create_err(crate::session_diagnostics::MutablyBorrowPin {
+            place,
+            span,
+            pin_place,
+            pin_span,
+        })
+    }
 }
 
 pub(crate) fn borrowed_data_escapes_closure<'tcx>(

compiler/rustc_borrowck/src/dataflow.rs

@@ -21,6 +21,7 @@ use crate::{BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext, plac
 // computed individually with `iterate_to_fixpoint`.
 pub(crate) struct Borrowck<'a, 'tcx> {
     pub(crate) borrows: Borrows<'a, 'tcx>,
+    pub(crate) pins: Pins<'a, 'tcx>,
     pub(crate) uninits: MaybeUninitializedPlaces<'a, 'tcx>,
     pub(crate) ever_inits: EverInitializedPlaces<'a, 'tcx>,
 }
@@ -33,6 +34,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> {
     fn bottom_value(&self, body: &mir::Body<'tcx>) -> Self::Domain {
         BorrowckDomain {
             borrows: self.borrows.bottom_value(body),
+            pinned_borrows: self.pins.bottom_value(body),
             uninits: self.uninits.bottom_value(body),
             ever_inits: self.ever_inits.bottom_value(body),
         }
@@ -50,6 +52,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> {
         loc: Location,
     ) {
         self.borrows.apply_early_statement_effect(&mut state.borrows, stmt, loc);
+        self.pins.apply_early_statement_effect(&mut state.pinned_borrows, stmt, loc);
         self.uninits.apply_early_statement_effect(&mut state.uninits, stmt, loc);
         self.ever_inits.apply_early_statement_effect(&mut state.ever_inits, stmt, loc);
     }
@@ -61,6 +64,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> {
         loc: Location,
     ) {
         self.borrows.apply_primary_statement_effect(&mut state.borrows, stmt, loc);
+        self.pins.apply_primary_statement_effect(&mut state.pinned_borrows, stmt, loc);
         self.uninits.apply_primary_statement_effect(&mut state.uninits, stmt, loc);
         self.ever_inits.apply_primary_statement_effect(&mut state.ever_inits, stmt, loc);
     }
@@ -72,6 +76,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> {
         loc: Location,
     ) {
         self.borrows.apply_early_terminator_effect(&mut state.borrows, term, loc);
+        self.pins.apply_early_terminator_effect(&mut state.pinned_borrows, term, loc);
         self.uninits.apply_early_terminator_effect(&mut state.uninits, term, loc);
         self.ever_inits.apply_early_terminator_effect(&mut state.ever_inits, term, loc);
     }
@@ -83,6 +88,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> {
         loc: Location,
     ) -> TerminatorEdges<'mir, 'tcx> {
         self.borrows.apply_primary_terminator_effect(&mut state.borrows, term, loc);
+        self.pins.apply_primary_terminator_effect(&mut state.pinned_borrows, term, loc);
         self.uninits.apply_primary_terminator_effect(&mut state.uninits, term, loc);
         self.ever_inits.apply_primary_terminator_effect(&mut state.ever_inits, term, loc);
 
@@ -116,6 +122,8 @@ where
     fn fmt_with(&self, ctxt: &C, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str("borrows: ")?;
         self.borrows.fmt_with(ctxt, f)?;
+        f.write_str(" pinned_borrows: ")?;
+        self.pinned_borrows.fmt_with(ctxt, f)?;
         f.write_str(" uninits: ")?;
         self.uninits.fmt_with(ctxt, f)?;
         f.write_str(" ever_inits: ")?;
@@ -134,6 +142,12 @@ where
             f.write_str("\n")?;
         }
 
+        if self.pinned_borrows != old.pinned_borrows {
+            f.write_str("pinned_borrows: ")?;
+            self.pinned_borrows.fmt_diff_with(&old.pinned_borrows, ctxt, f)?;
+            f.write_str("\n")?;
+        }
+
         if self.uninits != old.uninits {
             f.write_str("uninits: ")?;
             self.uninits.fmt_diff_with(&old.uninits, ctxt, f)?;
@@ -154,6 +168,7 @@ where
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) struct BorrowckDomain {
     pub(crate) borrows: BorrowsDomain,
+    pub(crate) pinned_borrows: BorrowsDomain,
     pub(crate) uninits: MaybeUninitializedPlacesDomain,
     pub(crate) ever_inits: EverInitializedPlacesDomain,
 }
@@ -164,6 +179,8 @@ rustc_index::newtype_index! {
     pub struct BorrowIndex {}
 }
 
+impl<C> DebugWithContext<C> for BorrowIndex {}
+
 /// `Borrows` stores the data used in the analyses that track the flow
 /// of borrows.
 ///
@@ -178,6 +195,19 @@ pub struct Borrows<'a, 'tcx> {
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
+/// `Pins` stores the data used in the analyses that track the flow
+/// of pins.
+///
+/// It uniquely identifies every pinned borrow by a
+/// `PinIndex`, and maps each such index to a `PinData`
+/// describing the pin. These indexes are used for representing the
+/// pins in compact bitvectors.
+pub(crate) struct Pins<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    body: &'a Body<'tcx>,
+    borrow_set: &'a BorrowSet<'tcx>,
+}
+
 struct OutOfScopePrecomputer<'a, 'tcx> {
     visited: DenseBitSet<mir::BasicBlock>,
     visit_stack: Vec<mir::BasicBlock>,
@@ -616,4 +646,169 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> {
     }
 }
 
-impl<C> DebugWithContext<C> for BorrowIndex {}
+impl<'a, 'tcx> Pins<'a, 'tcx> {
+    pub(crate) fn new(
+        tcx: TyCtxt<'tcx>,
+        body: &'a Body<'tcx>,
+        borrow_set: &'a BorrowSet<'tcx>,
+    ) -> Self {
+        Pins { tcx, body, borrow_set }
+    }
+
+    fn gen_pins_on_place(&self, state: &mut <Self as Analysis<'tcx>>::Domain, place: Place<'tcx>) {
+        self.borrow_set
+            .local_map
+            .get(&place.local)
+            .into_iter()
+            .flat_map(|bs| bs.iter())
+            .copied()
+            .filter(|&index| self.borrow_set[index].borrowed_place == place)
+            .for_each(|index| {
+                state.gen_(index);
+            });
+    }
+
+    /// Kill any pins whose original pinned place conflicts with `place`.
+    fn kill_pins_on_place(&self, state: &mut <Self as Analysis<'tcx>>::Domain, place: Place<'tcx>) {
+        debug!("kill_pins_on_place: place={:?}", place);
+
+        let other_pins_of_local = self
+            .borrow_set
+            .local_map
+            .get(&place.local)
+            .into_iter()
+            .flat_map(|bs| bs.iter())
+            .copied();
+
+        // If the place is a local with no projections, all pins of this
+        // local must be killed. This is purely an optimization so we don't have to call
+        // `places_conflict` for every pin.
+        if place.projection.is_empty() {
+            state.kill_all(other_pins_of_local);
+            return;
+        }
+
+        // By passing `PlaceConflictBias::NoOverlap`, we conservatively assume that any given
+        // pair of array indices are not equal, so that when `places_conflict` returns true, we
+        // will be assured that two places being compared definitely denotes the same sets of
+        // locations.
+        let definitely_conflicting_pins = other_pins_of_local.filter(|&i| {
+            places_conflict(
+                self.tcx,
+                self.body,
+                self.borrow_set[i].borrowed_place,
+                place,
+                PlaceConflictBias::NoOverlap,
+            )
+        });
+
+        state.kill_all(definitely_conflicting_pins);
+    }
+}
+
+/// Forward dataflow computation of the set of pins that are in scope at a particular location.
+/// - we gen the introduced pins
+/// - we kill pins on locals going out of scope
+/// - we kill pins when the pinned place is moved or overwritten
+impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Pins<'_, 'tcx> {
+    type Domain = BorrowsDomain;
+
+    const NAME: &'static str = "pins";
+
+    fn bottom_value(&self, _: &mir::Body<'tcx>) -> Self::Domain {
+        // bottom = nothing is pinned yet
+        MixedBitSet::new_empty(self.borrow_set.len())
+    }
+
+    fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut Self::Domain) {
+        // no pins have been created prior to function execution
+    }
+
+    fn apply_early_statement_effect(
+        &self,
+        state: &mut Self::Domain,
+        statement: &mir::Statement<'tcx>,
+        _location: Location,
+    ) {
+        // Kill pins early on reassignment/StorageDead so that the visitor
+        // (which runs after the early phase) sees the updated pin state.
+        // Note: we do NOT kill pins when the Pin result local goes out of scope
+        // (kill_pins_by_pin_local), because a pinned place stays pinned until
+        // the place itself is reassigned.
+        match &statement.kind {
+            mir::StatementKind::Assign(box (lhs, _)) => {
+                self.kill_pins_on_place(state, *lhs);
+            }
+            mir::StatementKind::StorageDead(local) => {
+                self.kill_pins_on_place(state, Place::from(*local));
+            }
+            _ => {}
+        }
+    }
+
+    fn apply_primary_statement_effect(
+        &self,
+        state: &mut Self::Domain,
+        stmt: &mir::Statement<'tcx>,
+        _location: Location,
+    ) {
+        match &stmt.kind {
+            mir::StatementKind::Assign(box (lhs, rhs)) => {
+                self.kill_pins_on_place(state, *lhs);
+
+                // Only user-written `&pin` borrows create long-lived pin facts.
+                if let mir::Rvalue::Ref(_, mir::BorrowKind::Pinned(_, kind), place) = rhs
+                    && matches!(*kind, mir::PinBorrowKind::Persistent)
+                {
+                    let place_ty = place.ty(self.body, self.tcx).ty;
+                    if !place_ty.is_unpin(self.tcx, self.body.typing_env(self.tcx)) {
+                        self.gen_pins_on_place(state, *place);
+                    }
+                }
+            }
+
+            mir::StatementKind::StorageDead(local) => {
+                // Kill all pins on locals that are going out of scope
+                self.kill_pins_on_place(state, Place::from(*local));
+            }
+
+            mir::StatementKind::FakeRead(..)
+            | mir::StatementKind::SetDiscriminant { .. }
+            | mir::StatementKind::StorageLive(..)
+            | mir::StatementKind::PlaceMention(..)
+            | mir::StatementKind::AscribeUserType(..)
+            | mir::StatementKind::Coverage(..)
+            | mir::StatementKind::Intrinsic(..)
+            | mir::StatementKind::ConstEvalCounter
+            | mir::StatementKind::BackwardIncompatibleDropHint { .. }
+            | mir::StatementKind::Nop => {}
+        }
+    }
+
+    fn apply_early_terminator_effect(
+        &self,
+        _state: &mut Self::Domain,
+        _terminator: &mir::Terminator<'tcx>,
+        _location: Location,
+    ) {
+        // No early terminator effects for pins
+    }
+
+    fn apply_primary_terminator_effect<'mir>(
+        &self,
+        state: &mut Self::Domain,
+        terminator: &'mir mir::Terminator<'tcx>,
+        _location: Location,
+    ) -> TerminatorEdges<'mir, 'tcx> {
+        if let mir::TerminatorKind::InlineAsm { operands, .. } = &terminator.kind {
+            for op in operands {
+                if let mir::InlineAsmOperand::Out { place: Some(place), .. }
+                | mir::InlineAsmOperand::InOut { out_place: Some(place), .. } = *op
+                {
+                    self.kill_pins_on_place(state, place);
+                }
+            }
+        }
+        terminator.edges()
+    }
+}

compiler/rustc_borrowck/src/def_use.rs

@@ -56,11 +56,13 @@ pub(crate) fn categorize(context: PlaceContext) -> Option<DefUse> {
         PlaceContext::NonUse(NonUseContext::AscribeUserTy(_)) |
 
         PlaceContext::MutatingUse(MutatingUseContext::RawBorrow) |
+        PlaceContext::MutatingUse(MutatingUseContext::PinnedBorrow) |
         PlaceContext::NonMutatingUse(NonMutatingUseContext::RawBorrow) |
         PlaceContext::NonMutatingUse(NonMutatingUseContext::Inspect) |
         PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) |
         PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) |
-        PlaceContext::MutatingUse(MutatingUseContext::Retag) =>
+        PlaceContext::MutatingUse(MutatingUseContext::Retag) |
+        PlaceContext::NonMutatingUse(NonMutatingUseContext::PinnedBorrow) =>
             Some(DefUse::Use),
 
         ///////////////////////////////////////////////////////////////////////////