rust/kernel/init/macros.rs

   1 // SPDX-License-Identifier: Apache-2.0 OR MIT
   2
   3 //! This module provides the macros that actually implement the proc-macros `pin_data` and
   4 //! `pinned_drop`.
   5 //!
   6 //! These macros should never be called directly, since they expect their input to be
   7 //! in a certain format which is internal. Use the proc-macros instead.
   8 //!
   9 //! This architecture has been chosen because the kernel does not yet have access to `syn` which
  10 //! would make matters a lot easier for implementing these as proc-macros.
  11 //!
  12 //! # Macro expansion example
  13 //!
  14 //! This section is intended for readers trying to understand the macros in this module and the
  15 //! `pin_init!` macros from `init.rs`.
  16 //!
  17 //! We will look at the following example:
  18 //!
  19 //! ```rust,ignore
  20 //! # use kernel::init::*;
  21 //! # use core::pin::Pin;
  22 //! #[pin_data]
  23 //! #[repr(C)]
  24 //! struct Bar<T> {
  25 //!     #[pin]
  26 //!     t: T,
  27 //!     pub x: usize,
  28 //! }
  29 //!
  30 //! impl<T> Bar<T> {
  31 //!     fn new(t: T) -> impl PinInit<Self> {
  32 //!         pin_init!(Self { t, x: 0 })
  33 //!     }
  34 //! }
  35 //!
  36 //! #[pin_data(PinnedDrop)]
  37 //! struct Foo {
  38 //!     a: usize,
  39 //!     #[pin]
  40 //!     b: Bar<u32>,
  41 //! }
  42 //!
  43 //! #[pinned_drop]
  44 //! impl PinnedDrop for Foo {
  45 //!     fn drop(self: Pin<&mut Self>) {
  46 //!         println!("{self:p} is getting dropped.");
  47 //!     }
  48 //! }
  49 //!
  50 //! let a = 42;
  51 //! let initializer = pin_init!(Foo {
  52 //!     a,
  53 //!     b <- Bar::new(36),
  54 //! });
  55 //! ```
  56 //!
  57 //! This example includes the most common and important features of the pin-init API.
  58 //!
  59 //! Below you can find individual section about the different macro invocations. Here are some
  60 //! general things we need to take into account when designing macros:
  61 //! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
  62 //!   this ensures that the correct item is used, since users could define their own `mod core {}`
  63 //!   and then their own `panic!` inside to execute arbitrary code inside of our macro.
  64 //! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
  65 //!   expressions inside of an `unsafe` block in the macro, because this would allow users to do
  66 //!   `unsafe` operations without an associated `unsafe` block.
  67 //!
  68 //! ## `#[pin_data]` on `Bar`
  69 //!
  70 //! This macro is used to specify which fields are structurally pinned and which fields are not. It
  71 //! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
  72 //!
  73 //! Here is the definition of `Bar` from our example:
  74 //!
  75 //! ```rust,ignore
  76 //! # use kernel::init::*;
  77 //! #[pin_data]
  78 //! #[repr(C)]
  79 //! struct Bar<T> {
  80 //!     #[pin]
  81 //!     t: T,
  82 //!     pub x: usize,
  83 //! }
  84 //! ```
  85 //!
  86 //! This expands to the following code:
  87 //!
  88 //! ```rust,ignore
  89 //! // Firstly the normal definition of the struct, attributes are preserved:
  90 //! #[repr(C)]
  91 //! struct Bar<T> {
  92 //!     t: T,
  93 //!     pub x: usize,
  94 //! }
  95 //! // Then an anonymous constant is defined, this is because we do not want any code to access the
  96 //! // types that we define inside:
  97 //! const _: () = {
  98 //!     // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
  99 //!     // since we need to implement access functions for each field and thus need to know its
 100 //!     // type.
 101 //!     struct __ThePinData<T> {
 102 //!         __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
 103 //!     }
 104 //!     // We implement `Copy` for the pin-data struct, since all functions it defines will take
 105 //!     // `self` by value.
 106 //!     impl<T> ::core::clone::Clone for __ThePinData<T> {
 107 //!         fn clone(&self) -> Self {
 108 //!             *self
 109 //!         }
 110 //!     }
 111 //!     impl<T> ::core::marker::Copy for __ThePinData<T> {}
 112 //!     // For every field of `Bar`, the pin-data struct will define a function with the same name
 113 //!     // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
 114 //!     // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
 115 //!     // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
 116 //!     #[allow(dead_code)]
 117 //!     impl<T> __ThePinData<T> {
 118 //!         unsafe fn t<E>(
 119 //!             self,
 120 //!             slot: *mut T,
 121 //!             // Since `t` is `#[pin]`, this is `PinInit`.
 122 //!             init: impl ::kernel::init::PinInit<T, E>,
 123 //!         ) -> ::core::result::Result<(), E> {
 124 //!             unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
 125 //!         }
 126 //!         pub unsafe fn x<E>(
 127 //!             self,
 128 //!             slot: *mut usize,
 129 //!             // Since `x` is not `#[pin]`, this is `Init`.
 130 //!             init: impl ::kernel::init::Init<usize, E>,
 131 //!         ) -> ::core::result::Result<(), E> {
 132 //!             unsafe { ::kernel::init::Init::__init(init, slot) }
 133 //!         }
 134 //!     }
 135 //!     // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
 136 //!     // that we constructed above.
 137 //!     unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
 138 //!         type PinData = __ThePinData<T>;
 139 //!         unsafe fn __pin_data() -> Self::PinData {
 140 //!             __ThePinData {
 141 //!                 __phantom: ::core::marker::PhantomData,
 142 //!             }
 143 //!         }
 144 //!     }
 145 //!     // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
 146 //!     // struct. This is important to ensure that no user can implement a rouge `__pin_data`
 147 //!     // function without using `unsafe`.
 148 //!     unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
 149 //!         type Datee = Bar<T>;
 150 //!     }
 151 //!     // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
 152 //!     // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
 153 //!     // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
 154 //!     // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
 155 //!     // for two reasons:
 156 //!     // - `__phantom`: every generic must be used, since we cannot really know which generics
 157 //!     //   are used, we declere all and then use everything here once.
 158 //!     // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
 159 //!     //   over it. The lifetime is needed to work around the limitation that trait bounds must
 160 //!     //   not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
 161 //!     //   unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
 162 //!     //   into accepting these bounds regardless.
 163 //!     #[allow(dead_code)]
 164 //!     struct __Unpin<'__pin, T> {
 165 //!         __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
 166 //!         __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
 167 //!         // Our only `#[pin]` field is `t`.
 168 //!         t: T,
 169 //!     }
 170 //!     #[doc(hidden)]
 171 //!     impl<'__pin, T>
 172 //!         ::core::marker::Unpin for Bar<T> where __Unpin<'__pin, T>: ::core::marker::Unpin {}
 173 //!     // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
 174 //!     // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
 175 //!     // UB with only safe code, so we disallow this by giving a trait implementation error using
 176 //!     // a direct impl and a blanket implementation.
 177 //!     trait MustNotImplDrop {}
 178 //!     // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
 179 //!     // (normally people want to know if a type has any kind of drop glue at all, here we want
 180 //!     // to know if it has any kind of custom drop glue, which is exactly what this bound does).
 181 //!     #[allow(drop_bounds)]
 182 //!     impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
 183 //!     impl<T> MustNotImplDrop for Bar<T> {}
 184 //!     // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
 185 //!     // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
 186 //!     // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
 187 //!     #[allow(non_camel_case_types)]
 188 //!     trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
 189 //!     impl<T: ::kernel::init::PinnedDrop>
 190 //!         UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
 191 //!     impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
 192 //! };
 193 //! ```
 194 //!
 195 //! ## `pin_init!` in `impl Bar`
 196 //!
 197 //! This macro creates an pin-initializer for the given struct. It requires that the struct is
 198 //! annotated by `#[pin_data]`.
 199 //!
 200 //! Here is the impl on `Bar` defining the new function:
 201 //!
 202 //! ```rust,ignore
 203 //! impl<T> Bar<T> {
 204 //!     fn new(t: T) -> impl PinInit<Self> {
 205 //!         pin_init!(Self { t, x: 0 })
 206 //!     }
 207 //! }
 208 //! ```
 209 //!
 210 //! This expands to the following code:
 211 //!
 212 //! ```rust,ignore
 213 //! impl<T> Bar<T> {
 214 //!     fn new(t: T) -> impl PinInit<Self> {
 215 //!         {
 216 //!             // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
 217 //!             // return type and shadow it later when we insert the arbitrary user code. That way
 218 //!             // there will be no possibility of returning without `unsafe`.
 219 //!             struct __InitOk;
 220 //!             // Get the pin-data type from the initialized type.
 221 //!             // - the function is unsafe, hence the unsafe block
 222 //!             // - we `use` the `HasPinData` trait in the block, it is only available in that
 223 //!             //   scope.
 224 //!             let data = unsafe {
 225 //!                 use ::kernel::init::__internal::HasPinData;
 226 //!                 Self::__pin_data()
 227 //!             };
 228 //!             // Use `data` to help with type inference, the closure supplied will have the type
 229 //!             // `FnOnce(*mut Self) -> Result<__InitOk, Infallible>`.
 230 //!             let init = ::kernel::init::__internal::PinData::make_closure::<
 231 //!                 _,
 232 //!                 __InitOk,
 233 //!                 ::core::convert::Infallible,
 234 //!             >(data, move |slot| {
 235 //!                 {
 236 //!                     // Shadow the structure so it cannot be used to return early. If a user
 237 //!                     // tries to write `return Ok(__InitOk)`, then they get a type error, since
 238 //!                     // that will refer to this struct instead of the one defined above.
 239 //!                     struct __InitOk;
 240 //!                     // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
 241 //!                     unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
 242 //!                     // Since initialization could fail later (not in this case, since the error
 243 //!                     // type is `Infallible`) we will need to drop this field if there is an
 244 //!                     // error later. This `DropGuard` will drop the field when it gets dropped
 245 //!                     // and has not yet been forgotten. We make a reference to it, so users
 246 //!                     // cannot `mem::forget` it from the initializer, since the name is the same
 247 //!                     // as the field (including hygiene).
 248 //!                     let t = &unsafe {
 249 //!                         ::kernel::init::__internal::DropGuard::new(
 250 //!                             ::core::addr_of_mut!((*slot).t),
 251 //!                         )
 252 //!                     };
 253 //!                     // Expansion of `x: 0,`:
 254 //!                     // Since this can be an arbitrary expression we cannot place it inside of
 255 //!                     // the `unsafe` block, so we bind it here.
 256 //!                     let x = 0;
 257 //!                     unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
 258 //!                     // We again create a `DropGuard`.
 259 //!                     let x = &unsafe {
 260 //!                         ::kernel::init::__internal::DropGuard::new(
 261 //!                             ::core::addr_of_mut!((*slot).x),
 262 //!                         )
 263 //!                     };
 264 //!
 265 //!                     // Here we use the type checker to ensure that every field has been
 266 //!                     // initialized exactly once, since this is `if false` it will never get
 267 //!                     // executed, but still type-checked.
 268 //!                     // Additionally we abuse `slot` to automatically infer the correct type for
 269 //!                     // the struct. This is also another check that every field is accessible
 270 //!                     // from this scope.
 271 //!                     #[allow(unreachable_code, clippy::diverging_sub_expression)]
 272 //!                     if false {
 273 //!                         unsafe {
 274 //!                             ::core::ptr::write(
 275 //!                                 slot,
 276 //!                                 Self {
 277 //!                                     // We only care about typecheck finding every field here,
 278 //!                                     // the expression does not matter, just conjure one using
 279 //!                                     // `panic!()`:
 280 //!                                     t: ::core::panic!(),
 281 //!                                     x: ::core::panic!(),
 282 //!                                 },
 283 //!                             );
 284 //!                         };
 285 //!                     }
 286 //!                     // Since initialization has successfully completed, we can now forget the
 287 //!                     // guards. This is not `mem::forget`, since we only have `&DropGuard`.
 288 //!                     unsafe { ::kernel::init::__internal::DropGuard::forget(t) };
 289 //!                     unsafe { ::kernel::init::__internal::DropGuard::forget(x) };
 290 //!                 }
 291 //!                 // We leave the scope above and gain access to the previously shadowed
 292 //!                 // `__InitOk` that we need to return.
 293 //!                 Ok(__InitOk)
 294 //!             });
 295 //!             // Change the return type from `__InitOk` to `()`.
 296 //!             let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> {
 297 //!                 init(slot).map(|__InitOk| ())
 298 //!             };
 299 //!             // Construct the initializer.
 300 //!             let init = unsafe {
 301 //!                 ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
 302 //!             };
 303 //!             init
 304 //!         }
 305 //!     }
 306 //! }
 307 //! ```
 308 //!
 309 //! ## `#[pin_data]` on `Foo`
 310 //!
 311 //! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
 312 //! differences/new things in the expansion of the `Foo` definition:
 313 //!
 314 //! ```rust,ignore
 315 //! #[pin_data(PinnedDrop)]
 316 //! struct Foo {
 317 //!     a: usize,
 318 //!     #[pin]
 319 //!     b: Bar<u32>,
 320 //! }
 321 //! ```
 322 //!
 323 //! This expands to the following code:
 324 //!
 325 //! ```rust,ignore
 326 //! struct Foo {
 327 //!     a: usize,
 328 //!     b: Bar<u32>,
 329 //! }
 330 //! const _: () = {
 331 //!     struct __ThePinData {
 332 //!         __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
 333 //!     }
 334 //!     impl ::core::clone::Clone for __ThePinData {
 335 //!         fn clone(&self) -> Self {
 336 //!             *self
 337 //!         }
 338 //!     }
 339 //!     impl ::core::marker::Copy for __ThePinData {}
 340 //!     #[allow(dead_code)]
 341 //!     impl __ThePinData {
 342 //!         unsafe fn b<E>(
 343 //!             self,
 344 //!             slot: *mut Bar<u32>,
 345 //!             init: impl ::kernel::init::PinInit<Bar<u32>, E>,
 346 //!         ) -> ::core::result::Result<(), E> {
 347 //!             unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
 348 //!         }
 349 //!         unsafe fn a<E>(
 350 //!             self,
 351 //!             slot: *mut usize,
 352 //!             init: impl ::kernel::init::Init<usize, E>,
 353 //!         ) -> ::core::result::Result<(), E> {
 354 //!             unsafe { ::kernel::init::Init::__init(init, slot) }
 355 //!         }
 356 //!     }
 357 //!     unsafe impl ::kernel::init::__internal::HasPinData for Foo {
 358 //!         type PinData = __ThePinData;
 359 //!         unsafe fn __pin_data() -> Self::PinData {
 360 //!             __ThePinData {
 361 //!                 __phantom: ::core::marker::PhantomData,
 362 //!             }
 363 //!         }
 364 //!     }
 365 //!     unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
 366 //!         type Datee = Foo;
 367 //!     }
 368 //!     #[allow(dead_code)]
 369 //!     struct __Unpin<'__pin> {
 370 //!         __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
 371 //!         __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
 372 //!         b: Bar<u32>,
 373 //!     }
 374 //!     #[doc(hidden)]
 375 //!     impl<'__pin> ::core::marker::Unpin for Foo where __Unpin<'__pin>: ::core::marker::Unpin {}
 376 //!     // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
 377 //!     // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
 378 //!     // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
 379 //!     impl ::core::ops::Drop for Foo {
 380 //!         fn drop(&mut self) {
 381 //!             // Since we are getting dropped, no one else has a reference to `self` and thus we
 382 //!             // can assume that we never move.
 383 //!             let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
 384 //!             // Create the unsafe token that proves that we are inside of a destructor, this
 385 //!             // type is only allowed to be created in a destructor.
 386 //!             let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
 387 //!             ::kernel::init::PinnedDrop::drop(pinned, token);
 388 //!         }
 389 //!     }
 390 //! };
 391 //! ```
 392 //!
 393 //! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
 394 //!
 395 //! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
 396 //! extra parameter that should not be used at all. The macro hides that parameter.
 397 //!
 398 //! Here is the `PinnedDrop` impl for `Foo`:
 399 //!
 400 //! ```rust,ignore
 401 //! #[pinned_drop]
 402 //! impl PinnedDrop for Foo {
 403 //!     fn drop(self: Pin<&mut Self>) {
 404 //!         println!("{self:p} is getting dropped.");
 405 //!     }
 406 //! }
 407 //! ```
 408 //!
 409 //! This expands to the following code:
 410 //!
 411 //! ```rust,ignore
 412 //! // `unsafe`, full path and the token parameter are added, everything else stays the same.
 413 //! unsafe impl ::kernel::init::PinnedDrop for Foo {
 414 //!     fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
 415 //!         println!("{self:p} is getting dropped.");
 416 //!     }
 417 //! }
 418 //! ```
 419 //!
 420 //! ## `pin_init!` on `Foo`
 421 //!
 422 //! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
 423 //! of `pin_init!` on `Foo`:
 424 //!
 425 //! ```rust,ignore
 426 //! let a = 42;
 427 //! let initializer = pin_init!(Foo {
 428 //!     a,
 429 //!     b <- Bar::new(36),
 430 //! });
 431 //! ```
 432 //!
 433 //! This expands to the following code:
 434 //!
 435 //! ```rust,ignore
 436 //! let a = 42;
 437 //! let initializer = {
 438 //!     struct __InitOk;
 439 //!     let data = unsafe {
 440 //!         use ::kernel::init::__internal::HasPinData;
 441 //!         Foo::__pin_data()
 442 //!     };
 443 //!     let init = ::kernel::init::__internal::PinData::make_closure::<
 444 //!         _,
 445 //!         __InitOk,
 446 //!         ::core::convert::Infallible,
 447 //!     >(data, move |slot| {
 448 //!         {
 449 //!             struct __InitOk;
 450 //!             unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
 451 //!             let a = &unsafe {
 452 //!                 ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
 453 //!             };
 454 //!             let b = Bar::new(36);
 455 //!             unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
 456 //!             let b = &unsafe {
 457 //!                 ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
 458 //!             };
 459 //!
 460 //!             #[allow(unreachable_code, clippy::diverging_sub_expression)]
 461 //!             if false {
 462 //!                 unsafe {
 463 //!                     ::core::ptr::write(
 464 //!                         slot,
 465 //!                         Foo {
 466 //!                             a: ::core::panic!(),
 467 //!                             b: ::core::panic!(),
 468 //!                         },
 469 //!                     );
 470 //!                 };
 471 //!             }
 472 //!             unsafe { ::kernel::init::__internal::DropGuard::forget(a) };
 473 //!             unsafe { ::kernel::init::__internal::DropGuard::forget(b) };
 474 //!         }
 475 //!         Ok(__InitOk)
 476 //!     });
 477 //!     let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> {
 478 //!         init(slot).map(|__InitOk| ())
 479 //!     };
 480 //!     let init = unsafe {
 481 //!         ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
 482 //!     };
 483 //!     init
 484 //! };
 485 //! ```
 486
 487 /// Creates a `unsafe impl<...> PinnedDrop for $type` block.
 488 ///
 489 /// See [`PinnedDrop`] for more information.
 490 #[doc(hidden)]
 491 #[macro_export]
 492 macro_rules! __pinned_drop {
 493     (
 494         @impl_sig($($impl_sig:tt)*),
 495         @impl_body(
 496             $(#[$($attr:tt)*])*
 497             fn drop($($sig:tt)*) {
 498                 $($inner:tt)*
 499             }
 500         ),
 501     ) => {
 502         unsafe $($impl_sig)* {
 503             // Inherit all attributes and the type/ident tokens for the signature.
 504             $(#[$($attr)*])*
 505             fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
 506                 $($inner)*
 507             }
 508         }
 509     }
 510 }
 511
 512 /// This macro first parses the struct definition such that it separates pinned and not pinned
 513 /// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
 514 #[doc(hidden)]
 515 #[macro_export]
 516 macro_rules! __pin_data {
 517     // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
 518     (parse_input:
 519         @args($($pinned_drop:ident)?),
 520         @sig(
 521             $(#[$($struct_attr:tt)*])*
 522             $vis:vis struct $name:ident
 523             $(where $($whr:tt)*)?
 524         ),
 525         @impl_generics($($impl_generics:tt)*),
 526         @ty_generics($($ty_generics:tt)*),
 527         @body({ $($fields:tt)* }),
 528     ) => {
 529         // We now use token munching to iterate through all of the fields. While doing this we
 530         // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
 531         // wants these to be structurally pinned. The rest of the fields are the
 532         // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
 533         // order to declare the struct.
 534         //
 535         // In this call we also put some explaining comments for the parameters.
 536         $crate::__pin_data!(find_pinned_fields:
 537             // Attributes on the struct itself, these will just be propagated to be put onto the
 538             // struct definition.
 539             @struct_attrs($(#[$($struct_attr)*])*),
 540             // The visibility of the struct.
 541             @vis($vis),
 542             // The name of the struct.
 543             @name($name),
 544             // The 'impl generics', the generics that will need to be specified on the struct inside
 545             // of an `impl<$ty_generics>` block.
 546             @impl_generics($($impl_generics)*),
 547             // The 'ty generics', the generics that will need to be specified on the impl blocks.
 548             @ty_generics($($ty_generics)*),
 549             // The where clause of any impl block and the declaration.
 550             @where($($($whr)*)?),
 551             // The remaining fields tokens that need to be processed.
 552             // We add a `,` at the end to ensure correct parsing.
 553             @fields_munch($($fields)* ,),
 554             // The pinned fields.
 555             @pinned(),
 556             // The not pinned fields.
 557             @not_pinned(),
 558             // All fields.
 559             @fields(),
 560             // The accumulator containing all attributes already parsed.
 561             @accum(),
 562             // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
 563             @is_pinned(),
 564             // The proc-macro argument, this should be `PinnedDrop` or ``.
 565             @pinned_drop($($pinned_drop)?),
 566         );
 567     };
 568     (find_pinned_fields:
 569         @struct_attrs($($struct_attrs:tt)*),
 570         @vis($vis:vis),
 571         @name($name:ident),
 572         @impl_generics($($impl_generics:tt)*),
 573         @ty_generics($($ty_generics:tt)*),
 574         @where($($whr:tt)*),
 575         // We found a PhantomPinned field, this should generally be pinned!
 576         @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
 577         @pinned($($pinned:tt)*),
 578         @not_pinned($($not_pinned:tt)*),
 579         @fields($($fields:tt)*),
 580         @accum($($accum:tt)*),
 581         // This field is not pinned.
 582         @is_pinned(),
 583         @pinned_drop($($pinned_drop:ident)?),
 584     ) => {
 585         ::core::compile_error!(concat!(
 586             "The field `",
 587             stringify!($field),
 588             "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
 589         ));
 590         $crate::__pin_data!(find_pinned_fields:
 591             @struct_attrs($($struct_attrs)*),
 592             @vis($vis),
 593             @name($name),
 594             @impl_generics($($impl_generics)*),
 595             @ty_generics($($ty_generics)*),
 596             @where($($whr)*),
 597             @fields_munch($($rest)*),
 598             @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
 599             @not_pinned($($not_pinned)*),
 600             @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
 601             @accum(),
 602             @is_pinned(),
 603             @pinned_drop($($pinned_drop)?),
 604         );
 605     };
 606     (find_pinned_fields:
 607         @struct_attrs($($struct_attrs:tt)*),
 608         @vis($vis:vis),
 609         @name($name:ident),
 610         @impl_generics($($impl_generics:tt)*),
 611         @ty_generics($($ty_generics:tt)*),
 612         @where($($whr:tt)*),
 613         // We reached the field declaration.
 614         @fields_munch($field:ident : $type:ty, $($rest:tt)*),
 615         @pinned($($pinned:tt)*),
 616         @not_pinned($($not_pinned:tt)*),
 617         @fields($($fields:tt)*),
 618         @accum($($accum:tt)*),
 619         // This field is pinned.
 620         @is_pinned(yes),
 621         @pinned_drop($($pinned_drop:ident)?),
 622     ) => {
 623         $crate::__pin_data!(find_pinned_fields:
 624             @struct_attrs($($struct_attrs)*),
 625             @vis($vis),
 626             @name($name),
 627             @impl_generics($($impl_generics)*),
 628             @ty_generics($($ty_generics)*),
 629             @where($($whr)*),
 630             @fields_munch($($rest)*),
 631             @pinned($($pinned)* $($accum)* $field: $type,),
 632             @not_pinned($($not_pinned)*),
 633             @fields($($fields)* $($accum)* $field: $type,),
 634             @accum(),
 635             @is_pinned(),
 636             @pinned_drop($($pinned_drop)?),
 637         );
 638     };
 639     (find_pinned_fields:
 640         @struct_attrs($($struct_attrs:tt)*),
 641         @vis($vis:vis),
 642         @name($name:ident),
 643         @impl_generics($($impl_generics:tt)*),
 644         @ty_generics($($ty_generics:tt)*),
 645         @where($($whr:tt)*),
 646         // We reached the field declaration.
 647         @fields_munch($field:ident : $type:ty, $($rest:tt)*),
 648         @pinned($($pinned:tt)*),
 649         @not_pinned($($not_pinned:tt)*),
 650         @fields($($fields:tt)*),
 651         @accum($($accum:tt)*),
 652         // This field is not pinned.
 653         @is_pinned(),
 654         @pinned_drop($($pinned_drop:ident)?),
 655     ) => {
 656         $crate::__pin_data!(find_pinned_fields:
 657             @struct_attrs($($struct_attrs)*),
 658             @vis($vis),
 659             @name($name),
 660             @impl_generics($($impl_generics)*),
 661             @ty_generics($($ty_generics)*),
 662             @where($($whr)*),
 663             @fields_munch($($rest)*),
 664             @pinned($($pinned)*),
 665             @not_pinned($($not_pinned)* $($accum)* $field: $type,),
 666             @fields($($fields)* $($accum)* $field: $type,),
 667             @accum(),
 668             @is_pinned(),
 669             @pinned_drop($($pinned_drop)?),
 670         );
 671     };
 672     (find_pinned_fields:
 673         @struct_attrs($($struct_attrs:tt)*),
 674         @vis($vis:vis),
 675         @name($name:ident),
 676         @impl_generics($($impl_generics:tt)*),
 677         @ty_generics($($ty_generics:tt)*),
 678         @where($($whr:tt)*),
 679         // We found the `#[pin]` attr.
 680         @fields_munch(#[pin] $($rest:tt)*),
 681         @pinned($($pinned:tt)*),
 682         @not_pinned($($not_pinned:tt)*),
 683         @fields($($fields:tt)*),
 684         @accum($($accum:tt)*),
 685         @is_pinned($($is_pinned:ident)?),
 686         @pinned_drop($($pinned_drop:ident)?),
 687     ) => {
 688         $crate::__pin_data!(find_pinned_fields:
 689             @struct_attrs($($struct_attrs)*),
 690             @vis($vis),
 691             @name($name),
 692             @impl_generics($($impl_generics)*),
 693             @ty_generics($($ty_generics)*),
 694             @where($($whr)*),
 695             @fields_munch($($rest)*),
 696             // We do not include `#[pin]` in the list of attributes, since it is not actually an
 697             // attribute that is defined somewhere.
 698             @pinned($($pinned)*),
 699             @not_pinned($($not_pinned)*),
 700             @fields($($fields)*),
 701             @accum($($accum)*),
 702             // Set this to `yes`.
 703             @is_pinned(yes),
 704             @pinned_drop($($pinned_drop)?),
 705         );
 706     };
 707     (find_pinned_fields:
 708         @struct_attrs($($struct_attrs:tt)*),
 709         @vis($vis:vis),
 710         @name($name:ident),
 711         @impl_generics($($impl_generics:tt)*),
 712         @ty_generics($($ty_generics:tt)*),
 713         @where($($whr:tt)*),
 714         // We reached the field declaration with visibility, for simplicity we only munch the
 715         // visibility and put it into `$accum`.
 716         @fields_munch($fvis:vis $field:ident $($rest:tt)*),
 717         @pinned($($pinned:tt)*),
 718         @not_pinned($($not_pinned:tt)*),
 719         @fields($($fields:tt)*),
 720         @accum($($accum:tt)*),
 721         @is_pinned($($is_pinned:ident)?),
 722         @pinned_drop($($pinned_drop:ident)?),
 723     ) => {
 724         $crate::__pin_data!(find_pinned_fields:
 725             @struct_attrs($($struct_attrs)*),
 726             @vis($vis),
 727             @name($name),
 728             @impl_generics($($impl_generics)*),
 729             @ty_generics($($ty_generics)*),
 730             @where($($whr)*),
 731             @fields_munch($field $($rest)*),
 732             @pinned($($pinned)*),
 733             @not_pinned($($not_pinned)*),
 734             @fields($($fields)*),
 735             @accum($($accum)* $fvis),
 736             @is_pinned($($is_pinned)?),
 737             @pinned_drop($($pinned_drop)?),
 738         );
 739     };
 740     (find_pinned_fields:
 741         @struct_attrs($($struct_attrs:tt)*),
 742         @vis($vis:vis),
 743         @name($name:ident),
 744         @impl_generics($($impl_generics:tt)*),
 745         @ty_generics($($ty_generics:tt)*),
 746         @where($($whr:tt)*),
 747         // Some other attribute, just put it into `$accum`.
 748         @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
 749         @pinned($($pinned:tt)*),
 750         @not_pinned($($not_pinned:tt)*),
 751         @fields($($fields:tt)*),
 752         @accum($($accum:tt)*),
 753         @is_pinned($($is_pinned:ident)?),
 754         @pinned_drop($($pinned_drop:ident)?),
 755     ) => {
 756         $crate::__pin_data!(find_pinned_fields:
 757             @struct_attrs($($struct_attrs)*),
 758             @vis($vis),
 759             @name($name),
 760             @impl_generics($($impl_generics)*),
 761             @ty_generics($($ty_generics)*),
 762             @where($($whr)*),
 763             @fields_munch($($rest)*),
 764             @pinned($($pinned)*),
 765             @not_pinned($($not_pinned)*),
 766             @fields($($fields)*),
 767             @accum($($accum)* #[$($attr)*]),
 768             @is_pinned($($is_pinned)?),
 769             @pinned_drop($($pinned_drop)?),
 770         );
 771     };
 772     (find_pinned_fields:
 773         @struct_attrs($($struct_attrs:tt)*),
 774         @vis($vis:vis),
 775         @name($name:ident),
 776         @impl_generics($($impl_generics:tt)*),
 777         @ty_generics($($ty_generics:tt)*),
 778         @where($($whr:tt)*),
 779         // We reached the end of the fields, plus an optional additional comma, since we added one
 780         // before and the user is also allowed to put a trailing comma.
 781         @fields_munch($(,)?),
 782         @pinned($($pinned:tt)*),
 783         @not_pinned($($not_pinned:tt)*),
 784         @fields($($fields:tt)*),
 785         @accum(),
 786         @is_pinned(),
 787         @pinned_drop($($pinned_drop:ident)?),
 788     ) => {
 789         // Declare the struct with all fields in the correct order.
 790         $($struct_attrs)*
 791         $vis struct $name <$($impl_generics)*>
 792         where $($whr)*
 793         {
 794             $($fields)*
 795         }
 796
 797         // We put the rest into this const item, because it then will not be accessible to anything
 798         // outside.
 799         const _: () = {
 800             // We declare this struct which will host all of the projection function for our type.
 801             // it will be invariant over all generic parameters which are inherited from the
 802             // struct.
 803             $vis struct __ThePinData<$($impl_generics)*>
 804             where $($whr)*
 805             {
 806                 __phantom: ::core::marker::PhantomData<
 807                     fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
 808                 >,
 809             }
 810
 811             impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
 812             where $($whr)*
 813             {
 814                 fn clone(&self) -> Self { *self }
 815             }
 816
 817             impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
 818             where $($whr)*
 819             {}
 820
 821             // Make all projection functions.
 822             $crate::__pin_data!(make_pin_data:
 823                 @pin_data(__ThePinData),
 824                 @impl_generics($($impl_generics)*),
 825                 @ty_generics($($ty_generics)*),
 826                 @where($($whr)*),
 827                 @pinned($($pinned)*),
 828                 @not_pinned($($not_pinned)*),
 829             );
 830
 831             // SAFETY: We have added the correct projection functions above to `__ThePinData` and
 832             // we also use the least restrictive generics possible.
 833             unsafe impl<$($impl_generics)*>
 834                 $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
 835             where $($whr)*
 836             {
 837                 type PinData = __ThePinData<$($ty_generics)*>;
 838
 839                 unsafe fn __pin_data() -> Self::PinData {
 840                     __ThePinData { __phantom: ::core::marker::PhantomData }
 841                 }
 842             }
 843
 844             unsafe impl<$($impl_generics)*>
 845                 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
 846             where $($whr)*
 847             {
 848                 type Datee = $name<$($ty_generics)*>;
 849             }
 850
 851             // This struct will be used for the unpin analysis. Since only structurally pinned
 852             // fields are relevant whether the struct should implement `Unpin`.
 853             #[allow(dead_code)]
 854             struct __Unpin <'__pin, $($impl_generics)*>
 855             where $($whr)*
 856             {
 857                 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
 858                 __phantom: ::core::marker::PhantomData<
 859                     fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
 860                 >,
 861                 // Only the pinned fields.
 862                 $($pinned)*
 863             }
 864
 865             #[doc(hidden)]
 866             impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
 867             where
 868                 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
 869                 $($whr)*
 870             {}
 871
 872             // We need to disallow normal `Drop` implementation, the exact behavior depends on
 873             // whether `PinnedDrop` was specified as the parameter.
 874             $crate::__pin_data!(drop_prevention:
 875                 @name($name),
 876                 @impl_generics($($impl_generics)*),
 877                 @ty_generics($($ty_generics)*),
 878                 @where($($whr)*),
 879                 @pinned_drop($($pinned_drop)?),
 880             );
 881         };
 882     };
 883     // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
 884     (drop_prevention:
 885         @name($name:ident),
 886         @impl_generics($($impl_generics:tt)*),
 887         @ty_generics($($ty_generics:tt)*),
 888         @where($($whr:tt)*),
 889         @pinned_drop(),
 890     ) => {
 891         // We prevent this by creating a trait that will be implemented for all types implementing
 892         // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
 893         // if it also implements `Drop`
 894         trait MustNotImplDrop {}
 895         #[allow(drop_bounds)]
 896         impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
 897         impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
 898         where $($whr)* {}
 899         // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
 900         // They might implement `PinnedDrop` correctly for the struct, but forget to give
 901         // `PinnedDrop` as the parameter to `#[pin_data]`.
 902         #[allow(non_camel_case_types)]
 903         trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
 904         impl<T: $crate::init::PinnedDrop>
 905             UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
 906         impl<$($impl_generics)*>
 907             UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
 908         where $($whr)* {}
 909     };
 910     // When `PinnedDrop` was specified we just implement `Drop` and delegate.
 911     (drop_prevention:
 912         @name($name:ident),
 913         @impl_generics($($impl_generics:tt)*),
 914         @ty_generics($($ty_generics:tt)*),
 915         @where($($whr:tt)*),
 916         @pinned_drop(PinnedDrop),
 917     ) => {
 918         impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
 919         where $($whr)*
 920         {
 921             fn drop(&mut self) {
 922                 // SAFETY: Since this is a destructor, `self` will not move after this function
 923                 // terminates, since it is inaccessible.
 924                 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
 925                 // SAFETY: Since this is a drop function, we can create this token to call the
 926                 // pinned destructor of this type.
 927                 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
 928                 $crate::init::PinnedDrop::drop(pinned, token);
 929             }
 930         }
 931     };
 932     // If some other parameter was specified, we emit a readable error.
 933     (drop_prevention:
 934         @name($name:ident),
 935         @impl_generics($($impl_generics:tt)*),
 936         @ty_generics($($ty_generics:tt)*),
 937         @where($($whr:tt)*),
 938         @pinned_drop($($rest:tt)*),
 939     ) => {
 940         compile_error!(
 941             "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
 942             stringify!($($rest)*),
 943         );
 944     };
 945     (make_pin_data:
 946         @pin_data($pin_data:ident),
 947         @impl_generics($($impl_generics:tt)*),
 948         @ty_generics($($ty_generics:tt)*),
 949         @where($($whr:tt)*),
 950         @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
 951         @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
 952     ) => {
 953         // For every field, we create a projection function according to its projection type. If a
 954         // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
 955         // structurally pinned, then it can be initialized via `Init`.
 956         //
 957         // The functions are `unsafe` to prevent accidentally calling them.
 958         #[allow(dead_code)]
 959         impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
 960         where $($whr)*
 961         {
 962             $(
 963                 $pvis unsafe fn $p_field<E>(
 964                     self,
 965                     slot: *mut $p_type,
 966                     init: impl $crate::init::PinInit<$p_type, E>,
 967                 ) -> ::core::result::Result<(), E> {
 968                     unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
 969                 }
 970             )*
 971             $(
 972                 $fvis unsafe fn $field<E>(
 973                     self,
 974                     slot: *mut $type,
 975                     init: impl $crate::init::Init<$type, E>,
 976                 ) -> ::core::result::Result<(), E> {
 977                     unsafe { $crate::init::Init::__init(init, slot) }
 978                 }
 979             )*
 980         }
 981     };
 982 }