diff options
Diffstat (limited to 'rust/kernel')
| -rw-r--r-- | rust/kernel/allocator.rs | 84 | ||||
| -rw-r--r-- | rust/kernel/error.rs | 4 | ||||
| -rw-r--r-- | rust/kernel/init.rs | 690 | ||||
| -rw-r--r-- | rust/kernel/init/__internal.rs | 39 | ||||
| -rw-r--r-- | rust/kernel/init/macros.rs | 519 | ||||
| -rw-r--r-- | rust/kernel/kunit.rs | 163 | ||||
| -rw-r--r-- | rust/kernel/lib.rs | 8 | ||||
| -rw-r--r-- | rust/kernel/prelude.rs | 2 | ||||
| -rw-r--r-- | rust/kernel/print.rs | 1 | ||||
| -rw-r--r-- | rust/kernel/str.rs | 4 | ||||
| -rw-r--r-- | rust/kernel/sync/arc.rs | 57 | ||||
| -rw-r--r-- | rust/kernel/sync/condvar.rs | 1 | ||||
| -rw-r--r-- | rust/kernel/sync/lock.rs | 7 | ||||
| -rw-r--r-- | rust/kernel/sync/lock/mutex.rs | 1 | ||||
| -rw-r--r-- | rust/kernel/sync/lock/spinlock.rs | 1 | ||||
| -rw-r--r-- | rust/kernel/task.rs | 2 | ||||
| -rw-r--r-- | rust/kernel/types.rs | 27 | ||||
| -rw-r--r-- | rust/kernel/workqueue.rs | 679 |
18 files changed, 1740 insertions, 549 deletions
diff --git a/rust/kernel/allocator.rs b/rust/kernel/allocator.rs index 9363b527be66..a8f3d5be1af1 100644 --- a/rust/kernel/allocator.rs +++ b/rust/kernel/allocator.rs @@ -41,9 +41,9 @@ unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: bindings::gf unsafe impl GlobalAlloc for KernelAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - // `krealloc()` is used instead of `kmalloc()` because the latter is - // an inline function and cannot be bound to as a result. - unsafe { bindings::krealloc(ptr::null(), layout.size(), bindings::GFP_KERNEL) as *mut u8 } + // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety + // requirement. + unsafe { krealloc_aligned(ptr::null_mut(), layout, bindings::GFP_KERNEL) } } unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { @@ -51,58 +51,38 @@ unsafe impl GlobalAlloc for KernelAllocator { bindings::kfree(ptr as *const core::ffi::c_void); } } -} - -#[global_allocator] -static ALLOCATOR: KernelAllocator = KernelAllocator; - -// `rustc` only generates these for some crate types. Even then, we would need -// to extract the object file that has them from the archive. For the moment, -// let's generate them ourselves instead. -// -// Note: Although these are *safe* functions, they are called by the compiler -// with parameters that obey the same `GlobalAlloc` function safety -// requirements: size and align should form a valid layout, and size is -// greater than 0. -// -// Note that `#[no_mangle]` implies exported too, nowadays. -#[no_mangle] -fn __rust_alloc(size: usize, align: usize) -> *mut u8 { - // SAFETY: See assumption above. - let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; - // SAFETY: `ptr::null_mut()` is null, per assumption above the size of `layout` is greater - // than 0. - unsafe { krealloc_aligned(ptr::null_mut(), layout, bindings::GFP_KERNEL) } -} + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + // SAFETY: + // - `new_size`, when rounded up to the nearest multiple of `layout.align()`, will not + // overflow `isize` by the function safety requirement. + // - `layout.align()` is a proper alignment (i.e. not zero and must be a power of two). + let layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + + // SAFETY: + // - `ptr` is either null or a pointer allocated by this allocator by the function safety + // requirement. + // - the size of `layout` is not zero because `new_size` is not zero by the function safety + // requirement. + unsafe { krealloc_aligned(ptr, layout, bindings::GFP_KERNEL) } + } -#[no_mangle] -fn __rust_dealloc(ptr: *mut u8, _size: usize, _align: usize) { - unsafe { bindings::kfree(ptr as *const core::ffi::c_void) }; + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety + // requirement. + unsafe { + krealloc_aligned( + ptr::null_mut(), + layout, + bindings::GFP_KERNEL | bindings::__GFP_ZERO, + ) + } + } } -#[no_mangle] -fn __rust_realloc(ptr: *mut u8, _old_size: usize, align: usize, new_size: usize) -> *mut u8 { - // SAFETY: See assumption above. - let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, align) }; - - // SAFETY: Per assumption above, `ptr` is allocated by `__rust_*` before, and the size of - // `new_layout` is greater than 0. - unsafe { krealloc_aligned(ptr, new_layout, bindings::GFP_KERNEL) } -} +#[global_allocator] +static ALLOCATOR: KernelAllocator = KernelAllocator; +// See <https://github.com/rust-lang/rust/pull/86844>. #[no_mangle] -fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8 { - // SAFETY: See assumption above. - let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; - - // SAFETY: `ptr::null_mut()` is null, per assumption above the size of `layout` is greater - // than 0. - unsafe { - krealloc_aligned( - ptr::null_mut(), - layout, - bindings::GFP_KERNEL | bindings::__GFP_ZERO, - ) - } -} +static __rust_no_alloc_shim_is_unstable: u8 = 0; diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 05fcab6abfe6..032b64543953 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -37,7 +37,7 @@ pub mod code { declare_err!(E2BIG, "Argument list too long."); declare_err!(ENOEXEC, "Exec format error."); declare_err!(EBADF, "Bad file number."); - declare_err!(ECHILD, "Exec format error."); + declare_err!(ECHILD, "No child processes."); declare_err!(EAGAIN, "Try again."); declare_err!(ENOMEM, "Out of memory."); declare_err!(EACCES, "Permission denied."); @@ -133,7 +133,7 @@ impl Error { /// Returns the error encoded as a pointer. #[allow(dead_code)] pub(crate) fn to_ptr<T>(self) -> *mut T { - // SAFETY: self.0 is a valid error due to its invariant. + // SAFETY: `self.0` is a valid error due to its invariant. unsafe { bindings::ERR_PTR(self.0.into()) as *mut _ } } diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index b4332a4ec1f4..65be9ae57b80 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -35,7 +35,7 @@ //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. //! //! ```rust -//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +//! # #![allow(clippy::disallowed_names)] //! use kernel::{prelude::*, sync::Mutex, new_mutex}; //! # use core::pin::Pin; //! #[pin_data] @@ -55,7 +55,7 @@ //! (or just the stack) to actually initialize a `Foo`: //! //! ```rust -//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +//! # #![allow(clippy::disallowed_names)] //! # use kernel::{prelude::*, sync::Mutex, new_mutex}; //! # use core::pin::Pin; //! # #[pin_data] @@ -86,7 +86,7 @@ //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: //! //! ```rust -//! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +//! # #![allow(clippy::disallowed_names)] //! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; //! #[pin_data] //! struct DriverData { @@ -120,14 +120,24 @@ //! `slot` gets called. //! //! ```rust -//! use kernel::{prelude::*, init}; +//! # #![allow(unreachable_pub, clippy::disallowed_names)] +//! use kernel::{prelude::*, init, types::Opaque}; //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; //! # mod bindings { +//! # #![allow(non_camel_case_types)] //! # pub struct foo; //! # pub unsafe fn init_foo(_ptr: *mut foo) {} //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } //! # } +//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround. +//! # trait FromErrno { +//! # fn from_errno(errno: core::ffi::c_int) -> Error { +//! # // Dummy error that can be constructed outside the `kernel` crate. +//! # Error::from(core::fmt::Error) +//! # } +//! # } +//! # impl FromErrno for Error {} //! /// # Invariants //! /// //! /// `foo` is always initialized @@ -158,7 +168,7 @@ //! if err != 0 { //! // Enabling has failed, first clean up the foo and then return the error. //! bindings::destroy_foo(Opaque::raw_get(foo)); -//! return Err(Error::from_kernel_errno(err)); +//! return Err(Error::from_errno(err)); //! } //! //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. @@ -202,11 +212,12 @@ use crate::{ error::{self, Error}, sync::UniqueArc, + types::{Opaque, ScopeGuard}, }; use alloc::boxed::Box; use core::{ alloc::AllocError, - cell::Cell, + cell::UnsafeCell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, @@ -225,9 +236,8 @@ pub mod macros; /// # Examples /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] -/// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; -/// # use macros::pin_data; +/// # #![allow(clippy::disallowed_names)] +/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; /// # use core::pin::Pin; /// #[pin_data] /// struct Foo { @@ -277,8 +287,8 @@ macro_rules! stack_pin_init { /// /// # Examples /// -/// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// ```rust,ignore +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; /// # use macros::pin_data; /// # use core::{alloc::AllocError, pin::Pin}; @@ -303,8 +313,8 @@ macro_rules! stack_pin_init { /// pr_info!("a: {}", &*foo.a.lock()); /// ``` /// -/// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// ```rust,ignore +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; /// # use macros::pin_data; /// # use core::{alloc::AllocError, pin::Pin}; @@ -356,7 +366,7 @@ macro_rules! stack_try_pin_init { /// The syntax is almost identical to that of a normal `struct` initializer: /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// #[pin_data] @@ -401,7 +411,7 @@ macro_rules! stack_try_pin_init { /// To create an initializer function, simply declare it like this: /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, prelude::*, init::*}; /// # use core::pin::Pin; /// # #[pin_data] @@ -428,7 +438,7 @@ macro_rules! stack_try_pin_init { /// Users of `Foo` can now create it like this: /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// # #[pin_data] @@ -456,7 +466,7 @@ macro_rules! stack_try_pin_init { /// They can also easily embed it into their own `struct`s: /// /// ```rust -/// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] +/// # #![allow(clippy::disallowed_names)] /// # use kernel::{init, pin_init, macros::pin_data, init::*}; /// # use core::pin::Pin; /// # #[pin_data] @@ -509,14 +519,17 @@ macro_rules! stack_try_pin_init { /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] /// pointer named `this` inside of the initializer. +/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the +/// struct, this initializes every field with 0 and then runs all initializers specified in the +/// body. This can only be done if [`Zeroable`] is implemented for the struct. /// /// For instance: /// /// ```rust -/// # use kernel::pin_init; -/// # use macros::pin_data; +/// # use kernel::{macros::{Zeroable, pin_data}, pin_init}; /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; /// #[pin_data] +/// #[derive(Zeroable)] /// struct Buf { /// // `ptr` points into `buf`. /// ptr: *mut u8, @@ -529,6 +542,10 @@ macro_rules! stack_try_pin_init { /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, /// pin: PhantomPinned, /// }); +/// pin_init!(Buf { +/// buf: [1; 64], +/// ..Zeroable::zeroed() +/// }); /// ``` /// /// [`try_pin_init!`]: kernel::try_pin_init @@ -540,11 +557,15 @@ macro_rules! pin_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_pin_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error(::core::convert::Infallible), + @data(PinData, use_data), + @has_data(HasPinData, __pin_data), + @construct_closure(pin_init_from_closure), + @munch_fields($($fields)*), ) }; } @@ -593,205 +614,31 @@ macro_rules! try_pin_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_pin_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)? ), @fields($($fields)*), @error($crate::error::Error), + @data(PinData, use_data), + @has_data(HasPinData, __pin_data), + @construct_closure(pin_init_from_closure), + @munch_fields($($fields)*), ) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - $crate::try_pin_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)? ), @fields($($fields)*), @error($err), + @data(PinData, use_data), + @has_data(HasPinData, __pin_data), + @construct_closure(pin_init_from_closure), + @munch_fields($($fields)*), ) }; - ( - @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), - @fields($($fields:tt)*), - @error($err:ty), - ) => {{ - // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return - // type and shadow it later when we insert the arbitrary user code. That way there will be - // no possibility of returning without `unsafe`. - struct __InitOk; - // Get the pin data from the supplied type. - let data = unsafe { - use $crate::init::__internal::HasPinData; - $t$(::<$($generics),*>)?::__pin_data() - }; - // Ensure that `data` really is of type `PinData` and help with type inference: - let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>( - data, - move |slot| { - { - // Shadow the structure so it cannot be used to return early. - struct __InitOk; - // Create the `this` so it can be referenced by the user inside of the - // expressions creating the individual fields. - $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? - // Initialize every field. - $crate::try_pin_init!(init_slot: - @data(data), - @slot(slot), - @munch_fields($($fields)*,), - ); - // We use unreachable code to ensure that all fields have been mentioned exactly - // once, this struct initializer will still be type-checked and complain with a - // very natural error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] - if false { - $crate::try_pin_init!(make_initializer: - @slot(slot), - @type_name($t), - @munch_fields($($fields)*,), - @acc(), - ); - } - // Forget all guards, since initialization was a success. - $crate::try_pin_init!(forget_guards: - @munch_fields($($fields)*,), - ); - } - Ok(__InitOk) - } - ); - let init = move |slot| -> ::core::result::Result<(), $err> { - init(slot).map(|__InitOk| ()) - }; - let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) }; - init - }}; - (init_slot: - @data($data:ident), - @slot($slot:ident), - @munch_fields($(,)?), - ) => { - // Endpoint of munching, no fields are left. - }; - (init_slot: - @data($data:ident), - @slot($slot:ident), - // In-place initialization syntax. - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - let $field = $val; - // Call the initializer. - // - // SAFETY: `slot` is valid, because we are inside of an initializer closure, we - // return when an error/panic occurs. - // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. - unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? }; - // Create the drop guard. - // - // We only give access to `&DropGuard`, so it cannot be forgotten via safe code. - // - // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) - }; - - $crate::try_pin_init!(init_slot: - @data($data), - @slot($slot), - @munch_fields($($rest)*), - ); - }; - (init_slot: - @data($data:ident), - @slot($slot:ident), - // Direct value init, this is safe for every field. - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - $(let $field = $val;)? - // Initialize the field. - // - // SAFETY: The memory at `slot` is uninitialized. - unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; - // Create the drop guard: - // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. - // - // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) - }; - - $crate::try_pin_init!(init_slot: - @data($data), - @slot($slot), - @munch_fields($($rest)*), - ); - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($(,)?), - @acc($($acc:tt)*), - ) => { - // Endpoint, nothing more to munch, create the initializer. - // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to - // get the correct type inference here: - unsafe { - ::core::ptr::write($slot, $t { - $($acc)* - }); - } - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - @acc($($acc:tt)*), - ) => { - $crate::try_pin_init!(make_initializer: - @slot($slot), - @type_name($t), - @munch_fields($($rest)*), - @acc($($acc)* $field: ::core::panic!(),), - ); - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - @acc($($acc:tt)*), - ) => { - $crate::try_pin_init!(make_initializer: - @slot($slot), - @type_name($t), - @munch_fields($($rest)*), - @acc($($acc)* $field: ::core::panic!(),), - ); - }; - (forget_guards: - @munch_fields($(,)?), - ) => { - // Munching finished. - }; - (forget_guards: - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_pin_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; - (forget_guards: - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_pin_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; } /// Construct an in-place initializer for `struct`s. @@ -816,11 +663,15 @@ macro_rules! init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error(::core::convert::Infallible), + @data(InitData, /*no use_data*/), + @has_data(HasInitData, __init_data), + @construct_closure(init_from_closure), + @munch_fields($($fields)*), ) } } @@ -841,7 +692,7 @@ macro_rules! init { /// # Examples /// /// ```rust -/// use kernel::{init::PinInit, error::Error, InPlaceInit}; +/// use kernel::{init::{PinInit, zeroed}, error::Error}; /// struct BigBuf { /// big: Box<[u8; 1024 * 1024 * 1024]>, /// small: [u8; 1024 * 1024], @@ -863,199 +714,31 @@ macro_rules! try_init { ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }) => { - $crate::try_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error($crate::error::Error), + @data(InitData, /*no use_data*/), + @has_data(HasInitData, __init_data), + @construct_closure(init_from_closure), + @munch_fields($($fields)*), ) }; ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { $($fields:tt)* }? $err:ty) => { - $crate::try_init!( + $crate::__init_internal!( @this($($this)?), @typ($t $(::<$($generics),*>)?), @fields($($fields)*), @error($err), + @data(InitData, /*no use_data*/), + @has_data(HasInitData, __init_data), + @construct_closure(init_from_closure), + @munch_fields($($fields)*), ) }; - ( - @this($($this:ident)?), - @typ($t:ident $(::<$($generics:ty),*>)?), - @fields($($fields:tt)*), - @error($err:ty), - ) => {{ - // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return - // type and shadow it later when we insert the arbitrary user code. That way there will be - // no possibility of returning without `unsafe`. - struct __InitOk; - // Get the init data from the supplied type. - let data = unsafe { - use $crate::init::__internal::HasInitData; - $t$(::<$($generics),*>)?::__init_data() - }; - // Ensure that `data` really is of type `InitData` and help with type inference: - let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>( - data, - move |slot| { - { - // Shadow the structure so it cannot be used to return early. - struct __InitOk; - // Create the `this` so it can be referenced by the user inside of the - // expressions creating the individual fields. - $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? - // Initialize every field. - $crate::try_init!(init_slot: - @slot(slot), - @munch_fields($($fields)*,), - ); - // We use unreachable code to ensure that all fields have been mentioned exactly - // once, this struct initializer will still be type-checked and complain with a - // very natural error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] - if false { - $crate::try_init!(make_initializer: - @slot(slot), - @type_name($t), - @munch_fields($($fields)*,), - @acc(), - ); - } - // Forget all guards, since initialization was a success. - $crate::try_init!(forget_guards: - @munch_fields($($fields)*,), - ); - } - Ok(__InitOk) - } - ); - let init = move |slot| -> ::core::result::Result<(), $err> { - init(slot).map(|__InitOk| ()) - }; - let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) }; - init - }}; - (init_slot: - @slot($slot:ident), - @munch_fields( $(,)?), - ) => { - // Endpoint of munching, no fields are left. - }; - (init_slot: - @slot($slot:ident), - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - let $field = $val; - // Call the initializer. - // - // SAFETY: `slot` is valid, because we are inside of an initializer closure, we - // return when an error/panic occurs. - unsafe { - $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; - } - // Create the drop guard. - // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. - // - // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) - }; - - $crate::try_init!(init_slot: - @slot($slot), - @munch_fields($($rest)*), - ); - }; - (init_slot: - @slot($slot:ident), - // Direct value init. - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - $(let $field = $val;)? - // Call the initializer. - // - // SAFETY: The memory at `slot` is uninitialized. - unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; - // Create the drop guard. - // - // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. - // - // SAFETY: We forget the guard later when initialization has succeeded. - let $field = &unsafe { - $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) - }; - - $crate::try_init!(init_slot: - @slot($slot), - @munch_fields($($rest)*), - ); - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields( $(,)?), - @acc($($acc:tt)*), - ) => { - // Endpoint, nothing more to munch, create the initializer. - // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to - // get the correct type inference here: - unsafe { - ::core::ptr::write($slot, $t { - $($acc)* - }); - } - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - @acc($($acc:tt)*), - ) => { - $crate::try_init!(make_initializer: - @slot($slot), - @type_name($t), - @munch_fields($($rest)*), - @acc($($acc)*$field: ::core::panic!(),), - ); - }; - (make_initializer: - @slot($slot:ident), - @type_name($t:ident), - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - @acc($($acc:tt)*), - ) => { - $crate::try_init!(make_initializer: - @slot($slot), - @type_name($t), - @munch_fields($($rest)*), - @acc($($acc)*$field: ::core::panic!(),), - ); - }; - (forget_guards: - @munch_fields($(,)?), - ) => { - // Munching finished. - }; - (forget_guards: - @munch_fields($field:ident <- $val:expr, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; - (forget_guards: - @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), - ) => { - unsafe { $crate::init::__internal::DropGuard::forget($field) }; - - $crate::try_init!(forget_guards: - @munch_fields($($rest)*), - ); - }; } /// A pin-initializer for the type `T`. @@ -1092,6 +775,79 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// deallocate. /// - `slot` will not move until it is dropped, i.e. it will be pinned. unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![allow(clippy::disallowed_names)] + /// use kernel::{types::Opaque, init::pin_init_from_closure}; + /// #[repr(C)] + /// struct RawFoo([u8; 16]); + /// extern { + /// fn init_foo(_: *mut RawFoo); + /// } + /// + /// #[pin_data] + /// struct Foo { + /// #[pin] + /// raw: Opaque<RawFoo>, + /// } + /// + /// impl Foo { + /// fn setup(self: Pin<&mut Self>) { + /// pr_info!("Setting up foo"); + /// } + /// } + /// + /// let foo = pin_init!(Foo { + /// raw <- unsafe { + /// Opaque::ffi_init(|s| { + /// init_foo(s); + /// }) + /// }, + /// }).pin_chain(|foo| { + /// foo.setup(); + /// Ok(()) + /// }); + /// ``` + fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> + where + F: FnOnce(Pin<&mut T>) -> Result<(), E>, + { + ChainPinInit(self, f, PhantomData) + } +} + +/// An initializer returned by [`PinInit::pin_chain`]. +pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); + +// SAFETY: The `__pinned_init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +// - considers `slot` pinned. +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E> +where + I: PinInit<T, E>, + F: FnOnce(Pin<&mut T>) -> Result<(), E>, +{ + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__pinned_init`. + unsafe { self.0.__pinned_init(slot)? }; + // SAFETY: The above call initialized `slot` and we still have unique access. + let val = unsafe { &mut *slot }; + // SAFETY: `slot` is considered pinned. + let val = unsafe { Pin::new_unchecked(val) }; + (self.1)(val).map_err(|e| { + // SAFETY: `slot` was initialized above. + unsafe { core::ptr::drop_in_place(slot) }; + e + }) + } } /// An initializer for `T`. @@ -1124,7 +880,7 @@ pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { /// /// [`Arc<T>`]: crate::sync::Arc #[must_use = "An initializer must be used in order to create its value."] -pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized { +pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { /// Initializes `slot`. /// /// # Safety @@ -1133,16 +889,73 @@ pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized { /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to /// deallocate. unsafe fn __init(self, slot: *mut T) -> Result<(), E>; + + /// First initializes the value using `self` then calls the function `f` with the initialized + /// value. + /// + /// If `f` returns an error the value is dropped and the initializer will forward the error. + /// + /// # Examples + /// + /// ```rust + /// # #![allow(clippy::disallowed_names)] + /// use kernel::{types::Opaque, init::{self, init_from_closure}}; + /// struct Foo { + /// buf: [u8; 1_000_000], + /// } + /// + /// impl Foo { + /// fn setup(&mut self) { + /// pr_info!("Setting up foo"); + /// } + /// } + /// + /// let foo = init!(Foo { + /// buf <- init::zeroed() + /// }).chain(|foo| { + /// foo.setup(); + /// Ok(()) + /// }); + /// ``` + fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> + where + F: FnOnce(&mut T) -> Result<(), E>, + { + ChainInit(self, f, PhantomData) + } +} + +/// An initializer returned by [`Init::chain`]. +pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); + +// SAFETY: The `__init` function is implemented such that it +// - returns `Ok(())` on successful initialization, +// - returns `Err(err)` on error and in this case `slot` will be dropped. +unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> +where + I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, +{ + unsafe fn __init(self, slot: *mut T) -> Result<(), E> { + // SAFETY: All requirements fulfilled since this function is `__init`. + unsafe { self.0.__pinned_init(slot)? }; + // SAFETY: The above call initialized `slot` and we still have unique access. + (self.1)(unsafe { &mut *slot }).map_err(|e| { + // SAFETY: `slot` was initialized above. + unsafe { core::ptr::drop_in_place(slot) }; + e + }) + } } -// SAFETY: Every in-place initializer can also be used as a pin-initializer. -unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I +// SAFETY: `__pinned_init` behaves exactly the same as `__init`. +unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> where I: Init<T, E>, + F: FnOnce(&mut T) -> Result<(), E>, { unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { - // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not - // require `slot` to not move after init. + // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. unsafe { self.__init(slot) } } } @@ -1194,6 +1007,93 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { unsafe { init_from_closure(|_| Ok(())) } } +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// use kernel::{error::Error, init::init_array_from_fn}; +/// let array: Box<[usize; 1_000]>= Box::init::<Error>(init_array_from_fn(|i| i)).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +pub fn init_array_from_fn<I, const N: usize, T, E>( + mut make_init: impl FnMut(usize) -> I, +) -> impl Init<[T; N], E> +where + I: Init<T, E>, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::<T>(); + // Counts the number of initialized elements and when dropped drops that many elements from + // `slot`. + let mut init_count = ScopeGuard::new_with_data(0, |i| { + // We now free every element that has been initialized before: + // SAFETY: The loop initialized exactly the values from 0..i and since we + // return `Err` below, the caller will consider the memory at `slot` as + // uninitialized. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; + }); + for i in 0..N { + let init = make_init(i); + // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + unsafe { init.__init(ptr) }?; + *init_count += 1; + } + init_count.dismiss(); + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { init_from_closure(init) } +} + +/// Initializes an array by initializing each element via the provided initializer. +/// +/// # Examples +/// +/// ```rust +/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex}; +/// let array: Arc<[Mutex<usize>; 1_000]>= +/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i))).unwrap(); +/// assert_eq!(array.len(), 1_000); +/// ``` +pub fn pin_init_array_from_fn<I, const N: usize, T, E>( + mut make_init: impl FnMut(usize) -> I, +) -> impl PinInit<[T; N], E> +where + I: PinInit<T, E>, +{ + let init = move |slot: *mut [T; N]| { + let slot = slot.cast::<T>(); + // Counts the number of initialized elements and when dropped drops that many elements from + // `slot`. + let mut init_count = ScopeGuard::new_with_data(0, |i| { + // We now free every element that has been initialized before: + // SAFETY: The loop initialized exactly the values from 0..i and since we + // return `Err` below, the caller will consider the memory at `slot` as + // uninitialized. + unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; + }); + for i in 0..N { + let init = make_init(i); + // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. + let ptr = unsafe { slot.add(i) }; + // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` + // requirements. + unsafe { init.__pinned_init(ptr) }?; + *init_count += 1; + } + init_count.dismiss(); + Ok(()) + }; + // SAFETY: The initializer above initializes every element of the array. On failure it drops + // any initialized elements and returns `Err`. + unsafe { pin_init_from_closure(init) } +} + // SAFETY: Every type can be initialized by-value. unsafe impl<T, E> Init<T, E> for T { unsafe fn __init(self, slot: *mut T) -> Result<(), E> { @@ -1202,6 +1102,13 @@ unsafe impl<T, E> Init<T, E> for T { } } +// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. +unsafe impl<T, E> PinInit<T, E> for T { + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + unsafe { self.__init(slot) } + } +} + /// Smart pointer that can initialize memory in-place. pub trait InPlaceInit<T>: Sized { /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this @@ -1390,6 +1297,11 @@ impl_zeroable! { // SAFETY: Type is allowed to take any value, including all zeros. {<T>} MaybeUninit<T>, + // SAFETY: Type is allowed to take any value, including all zeros. + {<T>} Opaque<T>, + + // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. + {<T: ?Sized + Zeroable>} UnsafeCell<T>, // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, diff --git a/rust/kernel/init/__internal.rs b/rust/kernel/init/__internal.rs index 44751fb62b51..db3372619ecd 100644 --- a/rust/kernel/init/__internal.rs +++ b/rust/kernel/init/__internal.rs @@ -13,7 +13,7 @@ use super::*; /// /// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns -type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; +pub(super) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>; /// This is the module-internal type implementing `PinInit` and `Init`. It is unsafe to create this /// type, since the closure needs to fulfill the same safety requirement as the @@ -32,6 +32,18 @@ where } } +// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the +// `__pinned_init` invariants. +unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E> +where + F: FnOnce(*mut T) -> Result<(), E>, +{ + #[inline] + unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { + (self.0)(slot) + } +} + /// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate /// the pin projections within the initializers. /// @@ -174,7 +186,6 @@ impl<T> StackInit<T> { /// Can be forgotten to prevent the drop. pub struct DropGuard<T: ?Sized> { ptr: *mut T, - do_drop: Cell<bool>, } impl<T: ?Sized> DropGuard<T> { @@ -190,32 +201,16 @@ impl<T: ?Sized> DropGuard<T> { /// - will not be dropped by any other means. #[inline] pub unsafe fn new(ptr: *mut T) -> Self { - Self { - ptr, - do_drop: Cell::new(true), - } - } - - /// Prevents this guard from dropping the supplied pointer. - /// - /// # Safety - /// - /// This function is unsafe in order to prevent safe code from forgetting this guard. It should - /// only be called by the macros in this module. - #[inline] - pub unsafe fn forget(&self) { - self.do_drop.set(false); + Self { ptr } } } impl<T: ?Sized> Drop for DropGuard<T> { #[inline] fn drop(&mut self) { - if self.do_drop.get() { - // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function - // ensuring that this operation is safe. - unsafe { ptr::drop_in_place(self.ptr) } - } + // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function + // ensuring that this operation is safe. + unsafe { ptr::drop_in_place(self.ptr) } } } diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs index 00aa4e956c0a..cb6e61b6c50b 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/kernel/init/macros.rs @@ -1,10 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT //! This module provides the macros that actually implement the proc-macros `pin_data` and -//! `pinned_drop`. +//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!` +//! macros. //! //! These macros should never be called directly, since they expect their input to be -//! in a certain format which is internal. Use the proc-macros instead. +//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in +//! safe code! Use the public facing macros instead. //! //! This architecture has been chosen because the kernel does not yet have access to `syn` which //! would make matters a lot easier for implementing these as proc-macros. @@ -43,7 +45,7 @@ //! #[pinned_drop] //! impl PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>) { -//! println!("{self:p} is getting dropped."); +//! pr_info!("{self:p} is getting dropped."); //! } //! } //! @@ -168,8 +170,10 @@ //! t: T, //! } //! #[doc(hidden)] -//! impl<'__pin, T> -//! ::core::marker::Unpin for Bar<T> where __Unpin<'__pin, T>: ::core::marker::Unpin {} +//! impl<'__pin, T> ::core::marker::Unpin for Bar<T> +//! where +//! __Unpin<'__pin, T>: ::core::marker::Unpin, +//! {} //! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users //! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to //! // UB with only safe code, so we disallow this by giving a trait implementation error using @@ -186,8 +190,9 @@ //! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. //! #[allow(non_camel_case_types)] //! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} -//! impl<T: ::kernel::init::PinnedDrop> -//! UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} +//! impl< +//! T: ::kernel::init::PinnedDrop, +//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} //! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {} //! }; //! ``` @@ -217,7 +222,7 @@ //! // return type and shadow it later when we insert the arbitrary user code. That way //! // there will be no possibility of returning without `unsafe`. //! struct __InitOk; -//! // Get the pin-data type from the initialized type. +//! // Get the data about fields from the supplied type. //! // - the function is unsafe, hence the unsafe block //! // - we `use` the `HasPinData` trait in the block, it is only available in that //! // scope. @@ -225,8 +230,7 @@ //! use ::kernel::init::__internal::HasPinData; //! Self::__pin_data() //! }; -//! // Use `data` to help with type inference, the closure supplied will have the type -//! // `FnOnce(*mut Self) -> Result<__InitOk, Infallible>`. +//! // Ensure that `data` really is of type `PinData` and help with type inference: //! let init = ::kernel::init::__internal::PinData::make_closure::< //! _, //! __InitOk, @@ -234,71 +238,75 @@ //! >(data, move |slot| { //! { //! // Shadow the structure so it cannot be used to return early. If a user -//! // tries to write `return Ok(__InitOk)`, then they get a type error, since -//! // that will refer to this struct instead of the one defined above. +//! // tries to write `return Ok(__InitOk)`, then they get a type error, +//! // since that will refer to this struct instead of the one defined +//! // above. //! struct __InitOk; //! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; -//! // Since initialization could fail later (not in this case, since the error -//! // type is `Infallible`) we will need to drop this field if there is an -//! // error later. This `DropGuard` will drop the field when it gets dropped -//! // and has not yet been forgotten. We make a reference to it, so users -//! // cannot `mem::forget` it from the initializer, since the name is the same -//! // as the field (including hygiene). -//! let t = &unsafe { -//! ::kernel::init::__internal::DropGuard::new( -//! ::core::addr_of_mut!((*slot).t), -//! ) +//! { +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; +//! } +//! // Since initialization could fail later (not in this case, since the +//! // error type is `Infallible`) we will need to drop this field if there +//! // is an error later. This `DropGuard` will drop the field when it gets +//! // dropped and has not yet been forgotten. +//! let t = unsafe { +//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) //! }; //! // Expansion of `x: 0,`: -//! // Since this can be an arbitrary expression we cannot place it inside of -//! // the `unsafe` block, so we bind it here. -//! let x = 0; -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; +//! // Since this can be an arbitrary expression we cannot place it inside +//! // of the `unsafe` block, so we bind it here. +//! { +//! let x = 0; +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; +//! } //! // We again create a `DropGuard`. -//! let x = &unsafe { -//! ::kernel::init::__internal::DropGuard::new( -//! ::core::addr_of_mut!((*slot).x), -//! ) +//! let x = unsafe { +//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) //! }; -//! +//! // Since initialization has successfully completed, we can now forget +//! // the guards. This is not `mem::forget`, since we only have +//! // `&DropGuard`. +//! ::core::mem::forget(x); +//! ::core::mem::forget(t); //! // Here we use the type checker to ensure that every field has been //! // initialized exactly once, since this is `if false` it will never get //! // executed, but still type-checked. -//! // Additionally we abuse `slot` to automatically infer the correct type for -//! // the struct. This is also another check that every field is accessible -//! // from this scope. +//! // Additionally we abuse `slot` to automatically infer the correct type +//! // for the struct. This is also another check that every field is +//! // accessible from this scope. //! #[allow(unreachable_code, clippy::diverging_sub_expression)] -//! if false { +//! let _ = || { //! unsafe { //! ::core::ptr::write( //! slot, //! Self { -//! // We only care about typecheck finding every field here, -//! // the expression does not matter, just conjure one using -//! // `panic!()`: +//! // We only care about typecheck finding every field +//! // here, the expression does not matter, just conjure +//! // one using `panic!()`: //! t: ::core::panic!(), //! x: ::core::panic!(), //! }, //! ); //! }; -//! } -//! // Since initialization has successfully completed, we can now forget the -//! // guards. This is not `mem::forget`, since we only have `&DropGuard`. -//! unsafe { ::kernel::init::__internal::DropGuard::forget(t) }; -//! unsafe { ::kernel::init::__internal::DropGuard::forget(x) }; +//! }; //! } //! // We leave the scope above and gain access to the previously shadowed //! // `__InitOk` that we need to return. //! Ok(__InitOk) //! }); //! // Change the return type from `__InitOk` to `()`. -//! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { +//! let init = move | +//! slot, +//! | -> ::core::result::Result<(), ::core::convert::Infallible> { //! init(slot).map(|__InitOk| ()) //! }; //! // Construct the initializer. //! let init = unsafe { -//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) +//! ::kernel::init::pin_init_from_closure::< +//! _, +//! ::core::convert::Infallible, +//! >(init) //! }; //! init //! } @@ -372,7 +380,10 @@ //! b: Bar<u32>, //! } //! #[doc(hidden)] -//! impl<'__pin> ::core::marker::Unpin for Foo where __Unpin<'__pin>: ::core::marker::Unpin {} +//! impl<'__pin> ::core::marker::Unpin for Foo +//! where +//! __Unpin<'__pin>: ::core::marker::Unpin, +//! {} //! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to //! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like //! // before, instead we implement `Drop` here and delegate to `PinnedDrop`. @@ -401,7 +412,7 @@ //! #[pinned_drop] //! impl PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>) { -//! println!("{self:p} is getting dropped."); +//! pr_info!("{self:p} is getting dropped."); //! } //! } //! ``` @@ -412,7 +423,7 @@ //! // `unsafe`, full path and the token parameter are added, everything else stays the same. //! unsafe impl ::kernel::init::PinnedDrop for Foo { //! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) { -//! println!("{self:p} is getting dropped."); +//! pr_info!("{self:p} is getting dropped."); //! } //! } //! ``` @@ -447,18 +458,21 @@ //! >(data, move |slot| { //! { //! struct __InitOk; -//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; -//! let a = &unsafe { +//! { +//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; +//! } +//! let a = unsafe { //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) //! }; -//! let b = Bar::new(36); +//! let init = Bar::new(36); //! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; -//! let b = &unsafe { +//! let b = unsafe { //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) //! }; -//! +//! ::core::mem::forget(b); +//! ::core::mem::forget(a); //! #[allow(unreachable_code, clippy::diverging_sub_expression)] -//! if false { +//! let _ = || { //! unsafe { //! ::core::ptr::write( //! slot, @@ -468,13 +482,13 @@ //! }, //! ); //! }; -//! } -//! unsafe { ::kernel::init::__internal::DropGuard::forget(a) }; -//! unsafe { ::kernel::init::__internal::DropGuard::forget(b) }; +//! }; //! } //! Ok(__InitOk) //! }); -//! let init = move |slot| -> ::core::result::Result<(), ::core::convert::Infallible> { +//! let init = move | +//! slot, +//! | -> ::core::result::Result<(), ::core::convert::Infallible> { //! init(slot).map(|__InitOk| ()) //! }; //! let init = unsafe { @@ -960,6 +974,7 @@ macro_rules! __pin_data { where $($whr)* { $( + $(#[$($p_attr)*])* $pvis unsafe fn $p_field<E>( self, slot: *mut $p_type, @@ -969,6 +984,7 @@ macro_rules! __pin_data { } )* $( + $(#[$($attr)*])* $fvis unsafe fn $field<E>( self, slot: *mut $type, @@ -980,3 +996,388 @@ macro_rules! __pin_data { } }; } + +/// The internal init macro. Do not call manually! +/// +/// This is called by the `{try_}{pin_}init!` macros with various inputs. +/// +/// This macro has multiple internal call configurations, these are always the very first ident: +/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros. +/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled. +/// - `init_slot`: recursively creates the code that initializes all fields in `slot`. +/// - `make_initializer`: recursively create the struct initializer that guarantees that every +/// field has been initialized exactly once. +#[doc(hidden)] +#[macro_export] +macro_rules! __init_internal { + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields(), + ) => { + $crate::__init_internal!(with_update_parsed: + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @zeroed(), // Nothing means default behavior. + ) + }; + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields(..Zeroable::zeroed()), + ) => { + $crate::__init_internal!(with_update_parsed: + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @zeroed(()), // `()` means zero all fields not mentioned. + ) + }; + ( + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @munch_fields($ignore:tt $($rest:tt)*), + ) => { + $crate::__init_internal!( + @this($($this)?), + @typ($t), + @fields($($fields)*), + @error($err), + @data($data, $($use_data)?), + @has_data($has_data, $get_data), + @construct_closure($construct_closure), + @munch_fields($($rest)*), + ) + }; + (with_update_parsed: + @this($($this:ident)?), + @typ($t:path), + @fields($($fields:tt)*), + @error($err:ty), + // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` + // case. + @data($data:ident, $($use_data:ident)?), + // `HasPinData` or `HasInitData`. + @has_data($has_data:ident, $get_data:ident), + // `pin_init_from_closure` or `init_from_closure`. + @construct_closure($construct_closure:ident), + @zeroed($($init_zeroed:expr)?), + ) => {{ + // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return + // type and shadow it later when we insert the arbitrary user code. That way there will be + // no possibility of returning without `unsafe`. + struct __InitOk; + // Get the data about fields from the supplied type. + let data = unsafe { + use $crate::init::__internal::$has_data; + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + ::kernel::macros::paste!($t::$get_data()) + }; + // Ensure that `data` really is of type `$data` and help with type inference: + let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>( + data, + move |slot| { + { + // Shadow the structure so it cannot be used to return early. + struct __InitOk; + // If `$init_zeroed` is present we should zero the slot now and not emit an + // error when fields are missing (since they will be zeroed). We also have to + // check that the type actually implements `Zeroable`. + $({ + fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {} + // Ensure that the struct is indeed `Zeroable`. + assert_zeroable(slot); + // SAFETY: The type implements `Zeroable` by the check above. + unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; + $init_zeroed // This will be `()` if set. + })? + // Create the `this` so it can be referenced by the user inside of the + // expressions creating the individual fields. + $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? + // Initialize every field. + $crate::__init_internal!(init_slot($($use_data)?): + @data(data), + @slot(slot), + @guards(), + @munch_fields($($fields)*,), + ); + // We use unreachable code to ensure that all fields have been mentioned exactly + // once, this struct initializer will still be type-checked and complain with a + // very natural error message if a field is forgotten/mentioned more than once. + #[allow(unreachable_code, clippy::diverging_sub_expression)] + let _ = || { + $crate::__init_internal!(make_initializer: + @slot(slot), + @type_name($t), + @munch_fields($($fields)*,), + @acc(), + ); + }; + } + Ok(__InitOk) + } + ); + let init = move |slot| -> ::core::result::Result<(), $err> { + init(slot).map(|__InitOk| ()) + }; + let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) }; + init + }}; + (init_slot($($use_data:ident)?): + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + @munch_fields($(..Zeroable::zeroed())? $(,)?), + ) => { + // Endpoint of munching, no fields are left. If execution reaches this point, all fields + // have been initialized. Therefore we can now dismiss the guards by forgetting them. + $(::core::mem::forget($guards);)* + }; + (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields. + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // In-place initialization syntax. + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + let init = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. + unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? }; + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + ::kernel::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [<$field>] = unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot($use_data): + @data($data), + @slot($slot), + @guards([<$field>], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (init_slot(): // No `use_data`, so we use `Init::__init` directly. + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // In-place initialization syntax. + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + ) => { + let init = $val; + // Call the initializer. + // + // SAFETY: `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + ::kernel::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [<$field>] = unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot(): + @data($data), + @slot($slot), + @guards([<$field>], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (init_slot($($use_data:ident)?): + @data($data:ident), + @slot($slot:ident), + @guards($($guards:ident,)*), + // Init by-value. + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + ) => { + { + $(let $field = $val;)? + // Initialize the field. + // + // SAFETY: The memory at `slot` is uninitialized. + unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; + } + // Create the drop guard: + // + // We rely on macro hygiene to make it impossible for users to access this local variable. + // We use `paste!` to create new hygiene for `$field`. + ::kernel::macros::paste! { + // SAFETY: We forget the guard later when initialization has succeeded. + let [<$field>] = unsafe { + $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) + }; + + $crate::__init_internal!(init_slot($($use_data)?): + @data($data), + @slot($slot), + @guards([<$field>], $($guards,)*), + @munch_fields($($rest)*), + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields(..Zeroable::zeroed() $(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. Since the users specified + // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have + // not been overwritten are thus zero and initialized. We still check that all fields are + // actually accessible by using the struct update syntax ourselves. + // We are inside of a closure that is never executed and thus we can abuse `slot` to + // get the correct type inference here: + #[allow(unused_assignments)] + unsafe { + let mut zeroed = ::core::mem::zeroed(); + // We have to use type inference here to make zeroed have the correct type. This does + // not get executed, so it has no effect. + ::core::ptr::write($slot, zeroed); + zeroed = ::core::mem::zeroed(); + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + ::kernel::macros::paste!( + ::core::ptr::write($slot, $t { + $($acc)* + ..zeroed + }); + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($(,)?), + @acc($($acc:tt)*), + ) => { + // Endpoint, nothing more to munch, create the initializer. + // Since we are in the closure that is never called, this will never get executed. + // We abuse `slot` to get the correct type inference here: + unsafe { + // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal + // information that is associated to already parsed fragments, so a path fragment + // cannot be used in this position. Doing the retokenization results in valid rust + // code. + ::kernel::macros::paste!( + ::core::ptr::write($slot, $t { + $($acc)* + }); + ); + } + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($field:ident <- $val:expr, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::__init_internal!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)* $field: ::core::panic!(),), + ); + }; + (make_initializer: + @slot($slot:ident), + @type_name($t:path), + @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), + @acc($($acc:tt)*), + ) => { + $crate::__init_internal!(make_initializer: + @slot($slot), + @type_name($t), + @munch_fields($($rest)*), + @acc($($acc)* $field: ::core::panic!(),), + ); + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __derive_zeroable { + (parse_input: + @sig( + $(#[$($struct_attr:tt)*])* + $vis:vis struct $name:ident + $(where $($whr:tt)*)? + ), + @impl_generics($($impl_generics:tt)*), + @ty_generics($($ty_generics:tt)*), + @body({ + $( + $(#[$($field_attr:tt)*])* + $field:ident : $field_ty:ty + ),* $(,)? + }), + ) => { + // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. + #[automatically_derived] + unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*> + where + $($($whr)*)? + {} + const _: () = { + fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {} + fn ensure_zeroable<$($impl_generics)*>() + where $($($whr)*)? + { + $(assert_zeroable::<$field_ty>();)* + } + }; + }; +} diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs new file mode 100644 index 000000000000..722655b2d62d --- /dev/null +++ b/rust/kernel/kunit.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! KUnit-based macros for Rust unit tests. +//! +//! C header: [`include/kunit/test.h`](../../../../../include/kunit/test.h) +//! +//! Reference: <https://docs.kernel.org/dev-tools/kunit/index.html> + +use core::{ffi::c_void, fmt}; + +/// Prints a KUnit error-level message. +/// +/// Public but hidden since it should only be used from KUnit generated code. +#[doc(hidden)] +pub fn err(args: fmt::Arguments<'_>) { + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we + // are passing. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + b"\x013%pA\0".as_ptr() as _, + &args as *const _ as *const c_void, + ); + } +} + +/// Prints a KUnit info-level message. +/// +/// Public but hidden since it should only be used from KUnit generated code. +#[doc(hidden)] +pub fn info(args: fmt::Arguments<'_>) { + // SAFETY: The format string is null-terminated and the `%pA` specifier matches the argument we + // are passing. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + b"\x016%pA\0".as_ptr() as _, + &args as *const _ as *const c_void, + ); + } +} + +/// Asserts that a boolean expression is `true` at runtime. +/// +/// Public but hidden since it should only be used from generated tests. +/// +/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit +/// facilities. See [`assert!`] for more details. +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_assert { + ($name:literal, $file:literal, $diff:expr, $condition:expr $(,)?) => { + 'out: { + // Do nothing if the condition is `true`. + if $condition { + break 'out; + } + + static FILE: &'static $crate::str::CStr = $crate::c_str!($file); + static LINE: i32 = core::line!() as i32 - $diff; + static CONDITION: &'static $crate::str::CStr = $crate::c_str!(stringify!($condition)); + + // SAFETY: FFI call without safety requirements. + let kunit_test = unsafe { $crate::bindings::kunit_get_current_test() }; + if kunit_test.is_null() { + // The assertion failed but this task is not running a KUnit test, so we cannot call + // KUnit, but at least print an error to the kernel log. This may happen if this + // macro is called from an spawned thread in a test (see + // `scripts/rustdoc_test_gen.rs`) or if some non-test code calls this macro by + // mistake (it is hidden to prevent that). + // + // This mimics KUnit's failed assertion format. + $crate::kunit::err(format_args!( + " # {}: ASSERTION FAILED at {FILE}:{LINE}\n", + $name + )); + $crate::kunit::err(format_args!( + " Expected {CONDITION} to be true, but is false\n" + )); + $crate::kunit::err(format_args!( + " Failure not reported to KUnit since this is a non-KUnit task\n" + )); + break 'out; + } + + #[repr(transparent)] + struct Location($crate::bindings::kunit_loc); + + #[repr(transparent)] + struct UnaryAssert($crate::bindings::kunit_unary_assert); + + // SAFETY: There is only a static instance and in that one the pointer field points to + // an immutable C string. + unsafe impl Sync for Location {} + + // SAFETY: There is only a static instance and in that one the pointer field points to + // an immutable C string. + unsafe impl Sync for UnaryAssert {} + + static LOCATION: Location = Location($crate::bindings::kunit_loc { + file: FILE.as_char_ptr(), + line: LINE, + }); + static ASSERTION: UnaryAssert = UnaryAssert($crate::bindings::kunit_unary_assert { + assert: $crate::bindings::kunit_assert {}, + condition: CONDITION.as_char_ptr(), + expected_true: true, + }); + + // SAFETY: + // - FFI call. + // - The `kunit_test` pointer is valid because we got it from + // `kunit_get_current_test()` and it was not null. This means we are in a KUnit + // test, and that the pointer can be passed to KUnit functions and assertions. + // - The string pointers (`file` and `condition` above) point to null-terminated + // strings since they are `CStr`s. + // - The function pointer (`format`) points to the proper function. + // - The pointers passed will remain valid since they point to `static`s. + // - The format string is allowed to be null. + // - There are, however, problems with this: first of all, this will end up stopping + // the thread, without running destructors. While that is problematic in itself, + // it is considered UB to have what is effectively a forced foreign unwind + // with `extern "C"` ABI. One could observe the stack that is now gone from + // another thread. We should avoid pinning stack variables to prevent library UB, + // too. For the moment, given that test failures are reported immediately before the + // next test runs, that test failures should be fixed and that KUnit is explicitly + // documented as not suitable for production environments, we feel it is reasonable. + unsafe { + $crate::bindings::__kunit_do_failed_assertion( + kunit_test, + core::ptr::addr_of!(LOCATION.0), + $crate::bindings::kunit_assert_type_KUNIT_ASSERTION, + core::ptr::addr_of!(ASSERTION.0.assert), + Some($crate::bindings::kunit_unary_assert_format), + core::ptr::null(), + ); + } + + // SAFETY: FFI call; the `test` pointer is valid because this hidden macro should only + // be called by the generated documentation tests which forward the test pointer given + // by KUnit. + unsafe { + $crate::bindings::__kunit_abort(kunit_test); + } + } + }; +} + +/// Asserts that two expressions are equal to each other (using [`PartialEq`]). +/// +/// Public but hidden since it should only be used from generated tests. +/// +/// Unlike the one in `core`, this one does not panic; instead, it is mapped to the KUnit +/// facilities. See [`assert!`] for more details. +#[doc(hidden)] +#[macro_export] +macro_rules! kunit_assert_eq { + ($name:literal, $file:literal, $diff:expr, $left:expr, $right:expr $(,)?) => {{ + // For the moment, we just forward to the expression assert because, for binary asserts, + // KUnit supports only a few types (e.g. integers). + $crate::kunit_assert!($name, $file, $diff, $left == $right); + }}; +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 85b261209977..e6aff80b521f 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,8 @@ #![feature(coerce_unsized)] #![feature(dispatch_from_dyn)] #![feature(new_uninit)] +#![feature(offset_of)] +#![feature(ptr_metadata)] #![feature(receiver_trait)] #![feature(unsize)] @@ -34,6 +36,8 @@ mod build_assert; pub mod error; pub mod init; pub mod ioctl; +#[cfg(CONFIG_KUNIT)] +pub mod kunit; pub mod prelude; pub mod print; mod static_assert; @@ -43,6 +47,7 @@ pub mod str; pub mod sync; pub mod task; pub mod types; +pub mod workqueue; #[doc(hidden)] pub use bindings; @@ -93,7 +98,4 @@ fn panic(info: &core::panic::PanicInfo<'_>) -> ! { pr_emerg!("{}\n", info); // SAFETY: FFI call. unsafe { bindings::BUG() }; - // Bindgen currently does not recognize `__noreturn` so `BUG` returns `()` - // instead of `!`. See <https://github.com/rust-lang/rust-bindgen/issues/2094>. - loop {} } diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index c28587d68ebc..ae21600970b3 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -18,7 +18,7 @@ pub use core::pin::Pin; pub use alloc::{boxed::Box, vec::Vec}; #[doc(no_inline)] -pub use macros::{module, pin_data, pinned_drop, vtable}; +pub use macros::{module, pin_data, pinned_drop, vtable, Zeroable}; pub use super::build_assert; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index 8009184bf6d7..f48926e3e9fe 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -399,6 +399,7 @@ macro_rules! pr_debug ( /// Mimics the interface of [`std::print!`]. See [`core::fmt`] and /// `alloc::format!` for information about the formatting syntax. /// +/// [`pr_info!`]: crate::pr_info! /// [`pr_cont`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_cont /// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html /// diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index c9dd3bf59e34..c41607b2e4fe 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -213,6 +213,7 @@ impl fmt::Display for CStr { /// /// ``` /// # use kernel::c_str; + /// # use kernel::fmt; /// # use kernel::str::CStr; /// # use kernel::str::CString; /// let penguin = c_str!("🐧"); @@ -241,6 +242,7 @@ impl fmt::Debug for CStr { /// /// ``` /// # use kernel::c_str; + /// # use kernel::fmt; /// # use kernel::str::CStr; /// # use kernel::str::CString; /// let penguin = c_str!("🐧"); @@ -529,7 +531,7 @@ impl fmt::Write for Formatter { /// # Examples /// /// ``` -/// use kernel::str::CString; +/// use kernel::{str::CString, fmt}; /// /// let s = CString::try_from_fmt(fmt!("{}{}{}", "abc", 10, 20)).unwrap(); /// assert_eq!(s.as_bytes_with_nul(), "abc1020\0".as_bytes()); diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 172f563976a9..77cdbcf7bd2e 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -24,13 +24,13 @@ use crate::{ }; use alloc::boxed::Box; use core::{ - alloc::AllocError, + alloc::{AllocError, Layout}, fmt, marker::{PhantomData, Unsize}, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, pin::Pin, - ptr::NonNull, + ptr::{NonNull, Pointee}, }; use macros::pin_data; @@ -73,6 +73,7 @@ mod std_vendor; /// assert_eq!(cloned.b, 20); /// /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed. +/// # Ok::<(), Error>(()) /// ``` /// /// Using `Arc<T>` as the type of `self`: @@ -98,6 +99,7 @@ mod std_vendor; /// let obj = Arc::try_new(Example { a: 10, b: 20 })?; /// obj.use_reference(); /// obj.take_over(); +/// # Ok::<(), Error>(()) /// ``` /// /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`: @@ -121,6 +123,7 @@ mod std_vendor; /// /// // `coerced` has type `Arc<dyn MyTrait>`. /// let coerced: Arc<dyn MyTrait> = obj; +/// # Ok::<(), Error>(()) /// ``` pub struct Arc<T: ?Sized> { ptr: NonNull<ArcInner<T>>, @@ -212,6 +215,48 @@ impl<T: ?Sized> Arc<T> { } } + /// Convert the [`Arc`] into a raw pointer. + /// + /// The raw pointer has ownership of the refcount that this Arc object owned. + pub fn into_raw(self) -> *const T { + let ptr = self.ptr.as_ptr(); + core::mem::forget(self); + // SAFETY: The pointer is valid. + unsafe { core::ptr::addr_of!((*ptr).data) } + } + + /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it + /// must not be called more than once for each previous call to [`Arc::into_raw`]. + pub unsafe fn from_raw(ptr: *const T) -> Self { + let refcount_layout = Layout::new::<bindings::refcount_t>(); + // SAFETY: The caller guarantees that the pointer is valid. + let val_layout = Layout::for_value(unsafe { &*ptr }); + // SAFETY: We're computing the layout of a real struct that existed when compiling this + // binary, so its layout is not so large that it can trigger arithmetic overflow. + let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; + + let metadata: <T as Pointee>::Metadata = core::ptr::metadata(ptr); + // SAFETY: The metadata of `T` and `ArcInner<T>` is the same because `ArcInner` is a struct + // with `T` as its last field. + // + // This is documented at: + // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>. + let metadata: <ArcInner<T> as Pointee>::Metadata = + unsafe { core::mem::transmute_copy(&metadata) }; + // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the + // pointer, since it originates from a previous call to `Arc::into_raw` and is still valid. + let ptr = unsafe { (ptr as *mut u8).sub(val_offset) as *mut () }; + let ptr = core::ptr::from_raw_parts_mut(ptr, metadata); + + // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the + // reference count held then will be owned by the new `Arc` object. + unsafe { Self::from_inner(NonNull::new_unchecked(ptr)) } + } + /// Returns an [`ArcBorrow`] from the given [`Arc`]. /// /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method @@ -299,7 +344,7 @@ impl<T: ?Sized> Drop for Arc<T> { // The count reached zero, we must free the memory. // // SAFETY: The pointer was initialised from the result of `Box::leak`. - unsafe { Box::from_raw(self.ptr.as_ptr()) }; + unsafe { drop(Box::from_raw(self.ptr.as_ptr())) }; } } } @@ -336,7 +381,7 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// # Example /// /// ``` -/// use crate::sync::{Arc, ArcBorrow}; +/// use kernel::sync::{Arc, ArcBorrow}; /// /// struct Example; /// @@ -349,12 +394,13 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// /// // Assert that both `obj` and `cloned` point to the same underlying object. /// assert!(core::ptr::eq(&*obj, &*cloned)); +/// # Ok::<(), Error>(()) /// ``` /// /// Using `ArcBorrow<T>` as the type of `self`: /// /// ``` -/// use crate::sync::{Arc, ArcBorrow}; +/// use kernel::sync::{Arc, ArcBorrow}; /// /// struct Example { /// a: u32, @@ -369,6 +415,7 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// /// let obj = Arc::try_new(Example { a: 10, b: 20 })?; /// obj.as_arc_borrow().use_reference(); +/// # Ok::<(), Error>(()) /// ``` pub struct ArcBorrow<'a, T: ?Sized + 'a> { inner: NonNull<ArcInner<T>>, diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index ed353399c4e5..b679b6f6dbeb 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -91,7 +91,6 @@ unsafe impl Sync for CondVar {} impl CondVar { /// Constructs a new condvar initialiser. - #[allow(clippy::new_ret_no_self)] pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { pin_init!(Self { _pin: PhantomPinned, diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index a2216325632d..f12a684bc957 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -72,8 +72,8 @@ pub unsafe trait Backend { /// A mutual exclusion primitive. /// -/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock backend -/// specified as the generic parameter `B`. +/// Exposes one of the kernel locking primitives. Which one is exposed depends on the lock +/// [`Backend`] specified as the generic parameter `B`. #[pin_data] pub struct Lock<T: ?Sized, B: Backend> { /// The kernel lock object. @@ -99,7 +99,6 @@ unsafe impl<T: ?Sized + Send, B: Backend> Sync for Lock<T, B> {} impl<T, B: Backend> Lock<T, B> { /// Constructs a new lock initialiser. - #[allow(clippy::new_ret_no_self)] pub fn new(t: T, name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> { pin_init!(Self { data: UnsafeCell::new(t), @@ -126,7 +125,7 @@ impl<T: ?Sized, B: Backend> Lock<T, B> { /// A lock guard. /// -/// Allows mutual exclusion primitives that implement the `Backend` trait to automatically unlock +/// Allows mutual exclusion primitives that implement the [`Backend`] trait to automatically unlock /// when a guard goes out of scope. It also provides a safe and convenient way to access the data /// protected by the lock. #[must_use = "the lock unlocks immediately when the guard is unused"] diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 923472f04af4..09276fedc091 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -63,6 +63,7 @@ macro_rules! new_mutex { /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); +/// # Ok::<(), Error>(()) /// ``` /// /// The following example shows how to use interior mutability to modify the contents of a struct diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 979b56464a4e..91eb2c9e9123 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -61,6 +61,7 @@ macro_rules! new_spinlock { /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); +/// # Ok::<(), Error>(()) /// ``` /// /// The following example shows how to use interior mutability to modify the contents of a struct diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index 7eda15e5f1b3..b2299bc7ac1f 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -82,7 +82,7 @@ impl Task { /// Returns a task reference for the currently executing task/thread. /// /// The recommended way to get the current task/thread is to use the - /// [`current`](crate::current) macro because it is safe. + /// [`current`] macro because it is safe. /// /// # Safety /// diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index d479f8da8f38..fdb778e65d79 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -6,7 +6,7 @@ use crate::init::{self, PinInit}; use alloc::boxed::Box; use core::{ cell::UnsafeCell, - marker::PhantomData, + marker::{PhantomData, PhantomPinned}, mem::MaybeUninit, ops::{Deref, DerefMut}, ptr::NonNull, @@ -91,7 +91,7 @@ impl ForeignOwnable for () { /// In the example below, we have multiple exit paths and we want to log regardless of which one is /// taken: /// ``` -/// # use kernel::ScopeGuard; +/// # use kernel::types::ScopeGuard; /// fn example1(arg: bool) { /// let _log = ScopeGuard::new(|| pr_info!("example1 completed\n")); /// @@ -109,7 +109,7 @@ impl ForeignOwnable for () { /// In the example below, we want to log the same message on all early exits but a different one on /// the main exit path: /// ``` -/// # use kernel::ScopeGuard; +/// # use kernel::types::ScopeGuard; /// fn example2(arg: bool) { /// let log = ScopeGuard::new(|| pr_info!("example2 returned early\n")); /// @@ -130,7 +130,7 @@ impl ForeignOwnable for () { /// In the example below, we need a mutable object (the vector) to be accessible within the log /// function, so we wrap it in the [`ScopeGuard`]: /// ``` -/// # use kernel::ScopeGuard; +/// # use kernel::types::ScopeGuard; /// fn example3(arg: bool) -> Result { /// let mut vec = /// ScopeGuard::new_with_data(Vec::new(), |v| pr_info!("vec had {} elements\n", v.len())); @@ -206,17 +206,26 @@ impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> { /// /// This is meant to be used with FFI objects that are never interpreted by Rust code. #[repr(transparent)] -pub struct Opaque<T>(MaybeUninit<UnsafeCell<T>>); +pub struct Opaque<T> { + value: UnsafeCell<MaybeUninit<T>>, + _pin: PhantomPinned, +} impl<T> Opaque<T> { /// Creates a new opaque value. pub const fn new(value: T) -> Self { - Self(MaybeUninit::new(UnsafeCell::new(value))) + Self { + value: UnsafeCell::new(MaybeUninit::new(value)), + _pin: PhantomPinned, + } } /// Creates an uninitialised value. pub const fn uninit() -> Self { - Self(MaybeUninit::uninit()) + Self { + value: UnsafeCell::new(MaybeUninit::uninit()), + _pin: PhantomPinned, + } } /// Creates a pin-initializer from the given initializer closure. @@ -240,7 +249,7 @@ impl<T> Opaque<T> { /// Returns a raw pointer to the opaque data. pub fn get(&self) -> *mut T { - UnsafeCell::raw_get(self.0.as_ptr()) + UnsafeCell::get(&self.value).cast::<T>() } /// Gets the value behind `this`. @@ -248,7 +257,7 @@ impl<T> Opaque<T> { /// This function is useful to get access to the value without creating intermediate /// references. pub const fn raw_get(this: *const Self) -> *mut T { - UnsafeCell::raw_get(this.cast::<UnsafeCell<T>>()) + UnsafeCell::raw_get(this.cast::<UnsafeCell<MaybeUninit<T>>>()).cast::<T>() } } diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs new file mode 100644 index 000000000000..b67fb1ba168e --- /dev/null +++ b/rust/kernel/workqueue.rs @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Work queues. +//! +//! This file has two components: The raw work item API, and the safe work item API. +//! +//! One pattern that is used in both APIs is the `ID` const generic, which exists to allow a single +//! type to define multiple `work_struct` fields. This is done by choosing an id for each field, +//! and using that id to specify which field you wish to use. (The actual value doesn't matter, as +//! long as you use different values for different fields of the same struct.) Since these IDs are +//! generic, they are used only at compile-time, so they shouldn't exist in the final binary. +//! +//! # The raw API +//! +//! The raw API consists of the `RawWorkItem` trait, where the work item needs to provide an +//! arbitrary function that knows how to enqueue the work item. It should usually not be used +//! directly, but if you want to, you can use it without using the pieces from the safe API. +//! +//! # The safe API +//! +//! The safe API is used via the `Work` struct and `WorkItem` traits. Furthermore, it also includes +//! a trait called `WorkItemPointer`, which is usually not used directly by the user. +//! +//! * The `Work` struct is the Rust wrapper for the C `work_struct` type. +//! * The `WorkItem` trait is implemented for structs that can be enqueued to a workqueue. +//! * The `WorkItemPointer` trait is implemented for the pointer type that points at a something +//! that implements `WorkItem`. +//! +//! ## Example +//! +//! This example defines a struct that holds an integer and can be scheduled on the workqueue. When +//! the struct is executed, it will print the integer. Since there is only one `work_struct` field, +//! we do not need to specify ids for the fields. +//! +//! ``` +//! use kernel::prelude::*; +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, Work, WorkItem}; +//! use kernel::{impl_has_work, new_work}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value: i32, +//! #[pin] +//! work: Work<MyStruct>, +//! } +//! +//! impl_has_work! { +//! impl HasWork<Self> for MyStruct { self.work } +//! } +//! +//! impl MyStruct { +//! fn new(value: i32) -> Result<Arc<Self>> { +//! Arc::pin_init(pin_init!(MyStruct { +//! value, +//! work <- new_work!("MyStruct::work"), +//! })) +//! } +//! } +//! +//! impl WorkItem for MyStruct { +//! type Pointer = Arc<MyStruct>; +//! +//! fn run(this: Arc<MyStruct>) { +//! pr_info!("The value is: {}", this.value); +//! } +//! } +//! +//! /// This method will enqueue the struct for execution on the system workqueue, where its value +//! /// will be printed. +//! fn print_later(val: Arc<MyStruct>) { +//! let _ = workqueue::system().enqueue(val); +//! } +//! ``` +//! +//! The following example shows how multiple `work_struct` fields can be used: +//! +//! ``` +//! use kernel::prelude::*; +//! use kernel::sync::Arc; +//! use kernel::workqueue::{self, Work, WorkItem}; +//! use kernel::{impl_has_work, new_work}; +//! +//! #[pin_data] +//! struct MyStruct { +//! value_1: i32, +//! value_2: i32, +//! #[pin] +//! work_1: Work<MyStruct, 1>, +//! #[pin] +//! work_2: Work<MyStruct, 2>, +//! } +//! +//! impl_has_work! { +//! impl HasWork<Self, 1> for MyStruct { self.work_1 } +//! impl HasWork<Self, 2> for MyStruct { self.work_2 } +//! } +//! +//! impl MyStruct { +//! fn new(value_1: i32, value_2: i32) -> Result<Arc<Self>> { +//! Arc::pin_init(pin_init!(MyStruct { +//! value_1, +//! value_2, +//! work_1 <- new_work!("MyStruct::work_1"), +//! work_2 <- new_work!("MyStruct::work_2"), +//! })) +//! } +//! } +//! +//! impl WorkItem<1> for MyStruct { +//! type Pointer = Arc<MyStruct>; +//! +//! fn run(this: Arc<MyStruct>) { +//! pr_info!("The value is: {}", this.value_1); +//! } +//! } +//! +//! impl WorkItem<2> for MyStruct { +//! type Pointer = Arc<MyStruct>; +//! +//! fn run(this: Arc<MyStruct>) { +//! pr_info!("The second value is: {}", this.value_2); +//! } +//! } +//! +//! fn print_1_later(val: Arc<MyStruct>) { +//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 1>(val); +//! } +//! +//! fn print_2_later(val: Arc<MyStruct>) { +//! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 2>(val); +//! } +//! ``` +//! +//! C header: [`include/linux/workqueue.h`](../../../../include/linux/workqueue.h) + +use crate::{bindings, prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; +use alloc::alloc::AllocError; +use alloc::boxed::Box; +use core::marker::PhantomData; +use core::pin::Pin; + +/// Creates a [`Work`] initialiser with the given name and a newly-created lock class. +#[macro_export] +macro_rules! new_work { + ($($name:literal)?) => { + $crate::workqueue::Work::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} + +/// A kernel work queue. +/// +/// Wraps the kernel's C `struct workqueue_struct`. +/// +/// It allows work items to be queued to run on thread pools managed by the kernel. Several are +/// always available, for example, `system`, `system_highpri`, `system_long`, etc. +#[repr(transparent)] +pub struct Queue(Opaque<bindings::workqueue_struct>); + +// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. +unsafe impl Send for Queue {} +// SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. +unsafe impl Sync for Queue {} + +impl Queue { + /// Use the provided `struct workqueue_struct` with Rust. + /// + /// # Safety + /// + /// The caller must ensure that the provided raw pointer is not dangling, that it points at a + /// valid workqueue, and that it remains valid until the end of 'a. + pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue { + // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The + // caller promises that the pointer is not dangling. + unsafe { &*(ptr as *const Queue) } + } + + /// Enqueues a work item. + /// + /// This may fail if the work item is already enqueued in a workqueue. + /// + /// The work item will be submitted using `WORK_CPU_UNBOUND`. + pub fn enqueue<W, const ID: u64>(&self, w: W) -> W::EnqueueOutput + where + W: RawWorkItem<ID> + Send + 'static, + { + let queue_ptr = self.0.get(); + + // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other + // `__enqueue` requirements are not relevant since `W` is `Send` and static. + // + // The call to `bindings::queue_work_on` will dereference the provided raw pointer, which + // is ok because `__enqueue` guarantees that the pointer is valid for the duration of this + // closure. + // + // Furthermore, if the C workqueue code accesses the pointer after this call to + // `__enqueue`, then the work item was successfully enqueued, and `bindings::queue_work_on` + // will have returned true. In this case, `__enqueue` promises that the raw pointer will + // stay valid until we call the function pointer in the `work_struct`, so the access is ok. + unsafe { + w.__enqueue(move |work_ptr| { + bindings::queue_work_on(bindings::WORK_CPU_UNBOUND as _, queue_ptr, work_ptr) + }) + } + } + + /// Tries to spawn the given function or closure as a work item. + /// + /// This method can fail because it allocates memory to store the work item. + pub fn try_spawn<T: 'static + Send + FnOnce()>(&self, func: T) -> Result<(), AllocError> { + let init = pin_init!(ClosureWork { + work <- new_work!("Queue::try_spawn"), + func: Some(func), + }); + + self.enqueue(Box::pin_init(init).map_err(|_| AllocError)?); + Ok(()) + } +} + +/// A helper type used in `try_spawn`. +#[pin_data] +struct ClosureWork<T> { + #[pin] + work: Work<ClosureWork<T>>, + func: Option<T>, +} + +impl<T> ClosureWork<T> { + fn project(self: Pin<&mut Self>) -> &mut Option<T> { + // SAFETY: The `func` field is not structurally pinned. + unsafe { &mut self.get_unchecked_mut().func } + } +} + +impl<T: FnOnce()> WorkItem for ClosureWork<T> { + type Pointer = Pin<Box<Self>>; + + fn run(mut this: Pin<Box<Self>>) { + if let Some(func) = this.as_mut().project().take() { + (func)() + } + } +} + +/// A raw work item. +/// +/// This is the low-level trait that is designed for being as general as possible. +/// +/// The `ID` parameter to this trait exists so that a single type can provide multiple +/// implementations of this trait. For example, if a struct has multiple `work_struct` fields, then +/// you will implement this trait once for each field, using a different id for each field. The +/// actual value of the id is not important as long as you use different ids for different fields +/// of the same struct. (Fields of different structs need not use different ids.) +/// +/// Note that the id is used only to select the right method to call during compilation. It wont be +/// part of the final executable. +/// +/// # Safety +/// +/// Implementers must ensure that any pointers passed to a `queue_work_on` closure by `__enqueue` +/// remain valid for the duration specified in the guarantees section of the documentation for +/// `__enqueue`. +pub unsafe trait RawWorkItem<const ID: u64> { + /// The return type of [`Queue::enqueue`]. + type EnqueueOutput; + + /// Enqueues this work item on a queue using the provided `queue_work_on` method. + /// + /// # Guarantees + /// + /// If this method calls the provided closure, then the raw pointer is guaranteed to point at a + /// valid `work_struct` for the duration of the call to the closure. If the closure returns + /// true, then it is further guaranteed that the pointer remains valid until someone calls the + /// function pointer stored in the `work_struct`. + /// + /// # Safety + /// + /// The provided closure may only return `false` if the `work_struct` is already in a workqueue. + /// + /// If the work item type is annotated with any lifetimes, then you must not call the function + /// pointer after any such lifetime expires. (Never calling the function pointer is okay.) + /// + /// If the work item type is not [`Send`], then the function pointer must be called on the same + /// thread as the call to `__enqueue`. + unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool; +} + +/// Defines the method that should be called directly when a work item is executed. +/// +/// This trait is implemented by `Pin<Box<T>>` and `Arc<T>`, and is mainly intended to be +/// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] +/// instead. The `run` method on this trait will usually just perform the appropriate +/// `container_of` translation and then call into the `run` method from the [`WorkItem`] trait. +/// +/// This trait is used when the `work_struct` field is defined using the [`Work`] helper. +/// +/// # Safety +/// +/// Implementers must ensure that [`__enqueue`] uses a `work_struct` initialized with the [`run`] +/// method of this trait as the function pointer. +/// +/// [`__enqueue`]: RawWorkItem::__enqueue +/// [`run`]: WorkItemPointer::run +pub unsafe trait WorkItemPointer<const ID: u64>: RawWorkItem<ID> { + /// Run this work item. + /// + /// # Safety + /// + /// The provided `work_struct` pointer must originate from a previous call to `__enqueue` where + /// the `queue_work_on` closure returned true, and the pointer must still be valid. + unsafe extern "C" fn run(ptr: *mut bindings::work_struct); +} + +/// Defines the method that should be called when this work item is executed. +/// +/// This trait is used when the `work_struct` field is defined using the [`Work`] helper. +pub trait WorkItem<const ID: u64 = 0> { + /// The pointer type that this struct is wrapped in. This will typically be `Arc<Self>` or + /// `Pin<Box<Self>>`. + type Pointer: WorkItemPointer<ID>; + + /// The method that should be called when this work item is executed. + fn run(this: Self::Pointer); +} + +/// Links for a work item. +/// +/// This struct contains a function pointer to the `run` function from the [`WorkItemPointer`] +/// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue. +/// +/// Wraps the kernel's C `struct work_struct`. +/// +/// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it. +#[repr(transparent)] +pub struct Work<T: ?Sized, const ID: u64 = 0> { + work: Opaque<bindings::work_struct>, + _inner: PhantomData<T>, +} + +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl<T: ?Sized, const ID: u64> Send for Work<T, ID> {} +// SAFETY: Kernel work items are usable from any thread. +// +// We do not need to constrain `T` since the work item does not actually contain a `T`. +unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {} + +impl<T: ?Sized, const ID: u64> Work<T, ID> { + /// Creates a new instance of [`Work`]. + #[inline] + #[allow(clippy::new_ret_no_self)] + pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> + where + T: WorkItem<ID>, + { + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as the work + // item function. + unsafe { + kernel::init::pin_init_from_closure(move |slot| { + let slot = Self::raw_get(slot); + bindings::init_work_with_key( + slot, + Some(T::Pointer::run), + false, + name.as_char_ptr(), + key.as_ptr(), + ); + Ok(()) + }) + } + } + + /// Get a pointer to the inner `work_struct`. + /// + /// # Safety + /// + /// The provided pointer must not be dangling and must be properly aligned. (But the memory + /// need not be initialized.) + #[inline] + pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct { + // SAFETY: The caller promises that the pointer is aligned and not dangling. + // + // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that + // the compiler does not complain that the `work` field is unused. + unsafe { Opaque::raw_get(core::ptr::addr_of!((*ptr).work)) } + } +} + +/// Declares that a type has a [`Work<T, ID>`] field. +/// +/// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro +/// like this: +/// +/// ```no_run +/// use kernel::impl_has_work; +/// use kernel::prelude::*; +/// use kernel::workqueue::Work; +/// +/// struct MyWorkItem { +/// work_field: Work<MyWorkItem, 1>, +/// } +/// +/// impl_has_work! { +/// impl HasWork<MyWorkItem, 1> for MyWorkItem { self.work_field } +/// } +/// ``` +/// +/// Note that since the `Work` type is annotated with an id, you can have several `work_struct` +/// fields by using a different id for each one. +/// +/// # Safety +/// +/// The [`OFFSET`] constant must be the offset of a field in Self of type [`Work<T, ID>`]. The methods on +/// this trait must have exactly the behavior that the definitions given below have. +/// +/// [`Work<T, ID>`]: Work +/// [`impl_has_work!`]: crate::impl_has_work +/// [`OFFSET`]: HasWork::OFFSET +pub unsafe trait HasWork<T, const ID: u64 = 0> { + /// The offset of the [`Work<T, ID>`] field. + /// + /// [`Work<T, ID>`]: Work + const OFFSET: usize; + + /// Returns the offset of the [`Work<T, ID>`] field. + /// + /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not Sized. + /// + /// [`Work<T, ID>`]: Work + /// [`OFFSET`]: HasWork::OFFSET + #[inline] + fn get_work_offset(&self) -> usize { + Self::OFFSET + } + + /// Returns a pointer to the [`Work<T, ID>`] field. + /// + /// # Safety + /// + /// The provided pointer must point at a valid struct of type `Self`. + /// + /// [`Work<T, ID>`]: Work + #[inline] + unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID> { + // SAFETY: The caller promises that the pointer is valid. + unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut Work<T, ID> } + } + + /// Returns a pointer to the struct containing the [`Work<T, ID>`] field. + /// + /// # Safety + /// + /// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`. + /// + /// [`Work<T, ID>`]: Work + #[inline] + unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self + where + Self: Sized, + { + // SAFETY: The caller promises that the pointer points at a field of the right type in the + // right kind of struct. + unsafe { (ptr as *mut u8).sub(Self::OFFSET) as *mut Self } + } +} + +/// Used to safely implement the [`HasWork<T, ID>`] trait. +/// +/// # Examples +/// +/// ``` +/// use kernel::impl_has_work; +/// use kernel::sync::Arc; +/// use kernel::workqueue::{self, Work}; +/// +/// struct MyStruct { +/// work_field: Work<MyStruct, 17>, +/// } +/// +/// impl_has_work! { +/// impl HasWork<MyStruct, 17> for MyStruct { self.work_field } +/// } +/// ``` +/// +/// [`HasWork<T, ID>`]: HasWork +#[macro_export] +macro_rules! impl_has_work { + ($(impl$(<$($implarg:ident),*>)? + HasWork<$work_type:ty $(, $id:tt)?> + for $self:ident $(<$($selfarg:ident),*>)? + { self.$field:ident } + )*) => {$( + // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right + // type. + unsafe impl$(<$($implarg),*>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self $(<$($selfarg),*>)? { + const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize; + + #[inline] + unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { + // SAFETY: The caller promises that the pointer is not dangling. + unsafe { + ::core::ptr::addr_of_mut!((*ptr).$field) + } + } + } + )*}; +} + +impl_has_work! { + impl<T> HasWork<Self> for ClosureWork<T> { self.work } +} + +unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T> +where + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. + let ptr = ptr as *mut Work<T, ID>; + // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. + let arc = unsafe { Arc::from_raw(ptr) }; + + T::run(arc) + } +} + +unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T> +where + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + type EnqueueOutput = Result<(), Self>; + + unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + // Casting between const and mut is not a problem as long as the pointer is a raw pointer. + let ptr = Arc::into_raw(self).cast_mut(); + + // SAFETY: Pointers into an `Arc` point at a valid value. + let work_ptr = unsafe { T::raw_get_work(ptr) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if queue_work_on(work_ptr) { + Ok(()) + } else { + // SAFETY: The work queue has not taken ownership of the pointer. + Err(unsafe { Arc::from_raw(ptr) }) + } + } +} + +unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<Box<T>> +where + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { + // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. + let ptr = ptr as *mut Work<T, ID>; + // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. + let ptr = unsafe { T::work_container_of(ptr) }; + // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. + let boxed = unsafe { Box::from_raw(ptr) }; + // SAFETY: The box was already pinned when it was enqueued. + let pinned = unsafe { Pin::new_unchecked(boxed) }; + + T::run(pinned) + } +} + +unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<Box<T>> +where + T: WorkItem<ID, Pointer = Self>, + T: HasWork<T, ID>, +{ + type EnqueueOutput = (); + + unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput + where + F: FnOnce(*mut bindings::work_struct) -> bool, + { + // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily + // remove the `Pin` wrapper. + let boxed = unsafe { Pin::into_inner_unchecked(self) }; + let ptr = Box::into_raw(boxed); + + // SAFETY: Pointers into a `Box` point at a valid value. + let work_ptr = unsafe { T::raw_get_work(ptr) }; + // SAFETY: `raw_get_work` returns a pointer to a valid value. + let work_ptr = unsafe { Work::raw_get(work_ptr) }; + + if !queue_work_on(work_ptr) { + // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a + // workqueue. + unsafe { ::core::hint::unreachable_unchecked() } + } + } +} + +/// Returns the system work queue (`system_wq`). +/// +/// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are +/// users which expect relatively short queue flush time. +/// +/// Callers shouldn't queue work items which can run for too long. +pub fn system() -> &'static Queue { + // SAFETY: `system_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_wq) } +} + +/// Returns the system high-priority work queue (`system_highpri_wq`). +/// +/// It is similar to the one returned by [`system`] but for work items which require higher +/// scheduling priority. +pub fn system_highpri() -> &'static Queue { + // SAFETY: `system_highpri_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_highpri_wq) } +} + +/// Returns the system work queue for potentially long-running work items (`system_long_wq`). +/// +/// It is similar to the one returned by [`system`] but may host long running work items. Queue +/// flushing might take relatively long. +pub fn system_long() -> &'static Queue { + // SAFETY: `system_long_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_long_wq) } +} + +/// Returns the system unbound work queue (`system_unbound_wq`). +/// +/// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items +/// are executed immediately as long as `max_active` limit is not reached and resources are +/// available. +pub fn system_unbound() -> &'static Queue { + // SAFETY: `system_unbound_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_unbound_wq) } +} + +/// Returns the system freezable work queue (`system_freezable_wq`). +/// +/// It is equivalent to the one returned by [`system`] except that it's freezable. +/// +/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work +/// items on the workqueue are drained and no new work item starts execution until thawed. +pub fn system_freezable() -> &'static Queue { + // SAFETY: `system_freezable_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_freezable_wq) } +} + +/// Returns the system power-efficient work queue (`system_power_efficient_wq`). +/// +/// It is inclined towards saving power and is converted to "unbound" variants if the +/// `workqueue.power_efficient` kernel parameter is specified; otherwise, it is similar to the one +/// returned by [`system`]. +pub fn system_power_efficient() -> &'static Queue { + // SAFETY: `system_power_efficient_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_power_efficient_wq) } +} + +/// Returns the system freezable power-efficient work queue (`system_freezable_power_efficient_wq`). +/// +/// It is similar to the one returned by [`system_power_efficient`] except that is freezable. +/// +/// A freezable workqueue participates in the freeze phase of the system suspend operations. Work +/// items on the workqueue are drained and no new work item starts execution until thawed. +pub fn system_freezable_power_efficient() -> &'static Queue { + // SAFETY: `system_freezable_power_efficient_wq` is a C global, always available. + unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) } +} |