diff options
Diffstat (limited to 'rust/kernel')
27 files changed, 2155 insertions, 323 deletions
diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs new file mode 100644 index 000000000000..531b5e471cb1 --- /dev/null +++ b/rust/kernel/alloc.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Extensions to the [`alloc`] crate. + +#[cfg(not(test))] +#[cfg(not(testlib))] +mod allocator; +pub mod box_ext; +pub mod vec_ext; + +/// Indicates an allocation error. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct AllocError; + +/// Flags to be used when allocating memory. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +/// +/// Values can be used from the [`flags`] module. +#[derive(Clone, Copy)] +pub struct Flags(u32); + +impl core::ops::BitOr for Flags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for Flags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for Flags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// Allocation flags. +/// +/// These are meant to be used in functions that can allocate memory. +pub mod flags { + use super::Flags; + + /// Zeroes out the allocated memory. + /// + /// This is normally or'd with other flags. + pub const __GFP_ZERO: Flags = Flags(bindings::__GFP_ZERO); + + /// Users can not sleep and need the allocation to succeed. + /// + /// A lower watermark is applied to allow access to "atomic reserves". The current + /// implementation doesn't support NMI and few other strict non-preemptive contexts (e.g. + /// raw_spin_lock). The same applies to [`GFP_NOWAIT`]. + pub const GFP_ATOMIC: Flags = Flags(bindings::GFP_ATOMIC); + + /// Typical for kernel-internal allocations. The caller requires ZONE_NORMAL or a lower zone + /// for direct access but can direct reclaim. + pub const GFP_KERNEL: Flags = Flags(bindings::GFP_KERNEL); + + /// The same as [`GFP_KERNEL`], except the allocation is accounted to kmemcg. + pub const GFP_KERNEL_ACCOUNT: Flags = Flags(bindings::GFP_KERNEL_ACCOUNT); + + /// Ror kernel allocations that should not stall for direct reclaim, start physical IO or + /// use any filesystem callback. It is very likely to fail to allocate memory, even for very + /// small allocations. + pub const GFP_NOWAIT: Flags = Flags(bindings::GFP_NOWAIT); +} diff --git a/rust/kernel/allocator.rs b/rust/kernel/alloc/allocator.rs index a8f3d5be1af1..229642960cd1 100644 --- a/rust/kernel/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -2,11 +2,10 @@ //! Allocator support. +use super::{flags::*, Flags}; use core::alloc::{GlobalAlloc, Layout}; use core::ptr; -use crate::bindings; - struct KernelAllocator; /// Calls `krealloc` with a proper size to alloc a new object aligned to `new_layout`'s alignment. @@ -15,13 +14,13 @@ struct KernelAllocator; /// /// - `ptr` can be either null or a pointer which has been allocated by this allocator. /// - `new_layout` must have a non-zero size. -unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: bindings::gfp_t) -> *mut u8 { +pub(crate) unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: Flags) -> *mut u8 { // Customized layouts from `Layout::from_size_align()` can have size < align, so pad first. let layout = new_layout.pad_to_align(); let mut size = layout.size(); - if layout.align() > bindings::BINDINGS_ARCH_SLAB_MINALIGN { + if layout.align() > bindings::ARCH_SLAB_MINALIGN { // The alignment requirement exceeds the slab guarantee, thus try to enlarge the size // to use the "power-of-two" size/alignment guarantee (see comments in `kmalloc()` for // more information). @@ -35,15 +34,15 @@ unsafe fn krealloc_aligned(ptr: *mut u8, new_layout: Layout, flags: bindings::gf // - `ptr` is either null or a pointer returned from a previous `k{re}alloc()` by the // function safety requirement. // - `size` is greater than 0 since it's either a `layout.size()` (which cannot be zero - // according to the function safety requirement) or a result from `next_power_of_two()`. - unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags) as *mut u8 } + // according to the function safety requirement) or a result from `next_power_of_two()`. + unsafe { bindings::krealloc(ptr as *const core::ffi::c_void, size, flags.0) as *mut u8 } } unsafe impl GlobalAlloc for KernelAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety // requirement. - unsafe { krealloc_aligned(ptr::null_mut(), layout, bindings::GFP_KERNEL) } + unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL) } } unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { @@ -64,19 +63,13 @@ unsafe impl GlobalAlloc for KernelAllocator { // requirement. // - the size of `layout` is not zero because `new_size` is not zero by the function safety // requirement. - unsafe { krealloc_aligned(ptr, layout, bindings::GFP_KERNEL) } + unsafe { krealloc_aligned(ptr, layout, GFP_KERNEL) } } unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { // SAFETY: `ptr::null_mut()` is null and `layout` has a non-zero size by the function safety // requirement. - unsafe { - krealloc_aligned( - ptr::null_mut(), - layout, - bindings::GFP_KERNEL | bindings::__GFP_ZERO, - ) - } + unsafe { krealloc_aligned(ptr::null_mut(), layout, GFP_KERNEL | __GFP_ZERO) } } } diff --git a/rust/kernel/alloc/box_ext.rs b/rust/kernel/alloc/box_ext.rs new file mode 100644 index 000000000000..829cb1c1cf9e --- /dev/null +++ b/rust/kernel/alloc/box_ext.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Extensions to [`Box`] for fallible allocations. + +use super::{AllocError, Flags}; +use alloc::boxed::Box; +use core::mem::MaybeUninit; + +/// Extensions to [`Box`]. +pub trait BoxExt<T>: Sized { + /// Allocates a new box. + /// + /// The allocation may fail, in which case an error is returned. + fn new(x: T, flags: Flags) -> Result<Self, AllocError>; + + /// Allocates a new uninitialised box. + /// + /// The allocation may fail, in which case an error is returned. + fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError>; +} + +impl<T> BoxExt<T> for Box<T> { + fn new(x: T, flags: Flags) -> Result<Self, AllocError> { + let b = <Self as BoxExt<_>>::new_uninit(flags)?; + Ok(Box::write(b, x)) + } + + #[cfg(any(test, testlib))] + fn new_uninit(_flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError> { + Ok(Box::new_uninit()) + } + + #[cfg(not(any(test, testlib)))] + fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>>, AllocError> { + let ptr = if core::mem::size_of::<MaybeUninit<T>>() == 0 { + core::ptr::NonNull::<_>::dangling().as_ptr() + } else { + let layout = core::alloc::Layout::new::<MaybeUninit<T>>(); + + // SAFETY: Memory is being allocated (first arg is null). The only other source of + // safety issues is sleeping on atomic context, which is addressed by klint. Lastly, + // the type is not a SZT (checked above). + let ptr = + unsafe { super::allocator::krealloc_aligned(core::ptr::null_mut(), layout, flags) }; + if ptr.is_null() { + return Err(AllocError); + } + + ptr.cast::<MaybeUninit<T>>() + }; + + // SAFETY: For non-zero-sized types, we allocate above using the global allocator. For + // zero-sized types, we use `NonNull::dangling`. + Ok(unsafe { Box::from_raw(ptr) }) + } +} diff --git a/rust/kernel/alloc/vec_ext.rs b/rust/kernel/alloc/vec_ext.rs new file mode 100644 index 000000000000..e9a81052728a --- /dev/null +++ b/rust/kernel/alloc/vec_ext.rs @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Extensions to [`Vec`] for fallible allocations. + +use super::{AllocError, Flags}; +use alloc::vec::Vec; +use core::ptr; + +/// Extensions to [`Vec`]. +pub trait VecExt<T>: Sized { + /// Creates a new [`Vec`] instance with at least the given capacity. + /// + /// # Examples + /// + /// ``` + /// let v = Vec::<u32>::with_capacity(20, GFP_KERNEL)?; + /// + /// assert!(v.capacity() >= 20); + /// # Ok::<(), Error>(()) + /// ``` + fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError>; + + /// Appends an element to the back of the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = Vec::new(); + /// v.push(1, GFP_KERNEL)?; + /// assert_eq!(&v, &[1]); + /// + /// v.push(2, GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 2]); + /// # Ok::<(), Error>(()) + /// ``` + fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError>; + + /// Pushes clones of the elements of slice into the [`Vec`] instance. + /// + /// # Examples + /// + /// ``` + /// let mut v = Vec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.extend_from_slice(&[20, 30, 40], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40]); + /// + /// v.extend_from_slice(&[50, 60], GFP_KERNEL)?; + /// assert_eq!(&v, &[1, 20, 30, 40, 50, 60]); + /// # Ok::<(), Error>(()) + /// ``` + fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> + where + T: Clone; + + /// Ensures that the capacity exceeds the length by at least `additional` elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = Vec::new(); + /// v.push(1, GFP_KERNEL)?; + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let cap = v.capacity(); + /// assert!(cap >= 10); + /// + /// v.reserve(10, GFP_KERNEL)?; + /// let new_cap = v.capacity(); + /// assert_eq!(new_cap, cap); + /// + /// # Ok::<(), Error>(()) + /// ``` + fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError>; +} + +impl<T> VecExt<T> for Vec<T> { + fn with_capacity(capacity: usize, flags: Flags) -> Result<Self, AllocError> { + let mut v = Vec::new(); + <Self as VecExt<_>>::reserve(&mut v, capacity, flags)?; + Ok(v) + } + + fn push(&mut self, v: T, flags: Flags) -> Result<(), AllocError> { + <Self as VecExt<_>>::reserve(self, 1, flags)?; + let s = self.spare_capacity_mut(); + s[0].write(v); + + // SAFETY: We just initialised the first spare entry, so it is safe to increase the length + // by 1. We also know that the new length is <= capacity because of the previous call to + // `reserve` above. + unsafe { self.set_len(self.len() + 1) }; + Ok(()) + } + + fn extend_from_slice(&mut self, other: &[T], flags: Flags) -> Result<(), AllocError> + where + T: Clone, + { + <Self as VecExt<_>>::reserve(self, other.len(), flags)?; + for (slot, item) in core::iter::zip(self.spare_capacity_mut(), other) { + slot.write(item.clone()); + } + + // SAFETY: We just initialised the `other.len()` spare entries, so it is safe to increase + // the length by the same amount. We also know that the new length is <= capacity because + // of the previous call to `reserve` above. + unsafe { self.set_len(self.len() + other.len()) }; + Ok(()) + } + + #[cfg(any(test, testlib))] + fn reserve(&mut self, additional: usize, _flags: Flags) -> Result<(), AllocError> { + Vec::reserve(self, additional); + Ok(()) + } + + #[cfg(not(any(test, testlib)))] + fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocError> { + let len = self.len(); + let cap = self.capacity(); + + if cap - len >= additional { + return Ok(()); + } + + if core::mem::size_of::<T>() == 0 { + // The capacity is already `usize::MAX` for SZTs, we can't go higher. + return Err(AllocError); + } + + // We know cap is <= `isize::MAX` because `Layout::array` fails if the resulting byte size + // is greater than `isize::MAX`. So the multiplication by two won't overflow. + let new_cap = core::cmp::max(cap * 2, len.checked_add(additional).ok_or(AllocError)?); + let layout = core::alloc::Layout::array::<T>(new_cap).map_err(|_| AllocError)?; + + let (old_ptr, len, cap) = destructure(self); + + // We need to make sure that `ptr` is either NULL or comes from a previous call to + // `krealloc_aligned`. A `Vec<T>`'s `ptr` value is not guaranteed to be NULL and might be + // dangling after being created with `Vec::new`. Instead, we can rely on `Vec<T>`'s capacity + // to be zero if no memory has been allocated yet. + let ptr = if cap == 0 { ptr::null_mut() } else { old_ptr }; + + // SAFETY: `ptr` is valid because it's either NULL or comes from a previous call to + // `krealloc_aligned`. We also verified that the type is not a ZST. + let new_ptr = unsafe { super::allocator::krealloc_aligned(ptr.cast(), layout, flags) }; + if new_ptr.is_null() { + // SAFETY: We are just rebuilding the existing `Vec` with no changes. + unsafe { rebuild(self, old_ptr, len, cap) }; + Err(AllocError) + } else { + // SAFETY: `ptr` has been reallocated with the layout for `new_cap` elements. New cap + // is greater than `cap`, so it continues to be >= `len`. + unsafe { rebuild(self, new_ptr.cast::<T>(), len, new_cap) }; + Ok(()) + } + } +} + +#[cfg(not(any(test, testlib)))] +fn destructure<T>(v: &mut Vec<T>) -> (*mut T, usize, usize) { + let mut tmp = Vec::new(); + core::mem::swap(&mut tmp, v); + let mut tmp = core::mem::ManuallyDrop::new(tmp); + let len = tmp.len(); + let cap = tmp.capacity(); + (tmp.as_mut_ptr(), len, cap) +} + +/// Rebuilds a `Vec` from a pointer, length, and capacity. +/// +/// # Safety +/// +/// The same as [`Vec::from_raw_parts`]. +#[cfg(not(any(test, testlib)))] +unsafe fn rebuild<T>(v: &mut Vec<T>, ptr: *mut T, len: usize, cap: usize) { + // SAFETY: The safety requirements from this function satisfy those of `from_raw_parts`. + let mut tmp = unsafe { Vec::from_raw_parts(ptr, len, cap) }; + core::mem::swap(&mut tmp, v); +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 032b64543953..55280ae9fe40 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -2,16 +2,12 @@ //! Kernel errors. //! -//! C header: [`include/uapi/asm-generic/errno-base.h`](../../../include/uapi/asm-generic/errno-base.h) +//! C header: [`include/uapi/asm-generic/errno-base.h`](srctree/include/uapi/asm-generic/errno-base.h) -use crate::str::CStr; +use crate::{alloc::AllocError, str::CStr}; -use alloc::{ - alloc::{AllocError, LayoutError}, - collections::TryReserveError, -}; +use alloc::alloc::LayoutError; -use core::convert::From; use core::fmt; use core::num::TryFromIntError; use core::str::Utf8Error; @@ -192,12 +188,6 @@ impl From<Utf8Error> for Error { } } -impl From<TryReserveError> for Error { - fn from(_: TryReserveError) -> Error { - code::ENOMEM - } -} - impl From<LayoutError> for Error { fn from(_: LayoutError) -> Error { code::ENOMEM @@ -264,13 +254,9 @@ pub fn to_result(err: core::ffi::c_int) -> Result { /// pdev: &mut PlatformDevice, /// index: u32, /// ) -> Result<*mut core::ffi::c_void> { -/// // SAFETY: FFI call. -/// unsafe { -/// from_err_ptr(bindings::devm_platform_ioremap_resource( -/// pdev.to_ptr(), -/// index, -/// )) -/// } +/// // SAFETY: `pdev` points to a valid platform device. There are no safety requirements +/// // on `index`. +/// from_err_ptr(unsafe { bindings::devm_platform_ioremap_resource(pdev.to_ptr(), index) }) /// } /// ``` // TODO: Remove `dead_code` marker once an in-kernel client is available. @@ -335,3 +321,7 @@ where Err(e) => T::from(e.to_errno() as i16), } } + +/// Error message for calling a default function of a [`#[vtable]`](macros::vtable) trait. +pub const VTABLE_DEFAULT_ERROR: &str = + "This function must not be called, see the #[vtable] documentation."; diff --git a/rust/kernel/init.rs b/rust/kernel/init.rs index 65be9ae57b80..68605b633e73 100644 --- a/rust/kernel/init.rs +++ b/rust/kernel/init.rs @@ -36,7 +36,7 @@ //! //! ```rust //! # #![allow(clippy::disallowed_names)] -//! use kernel::{prelude::*, sync::Mutex, new_mutex}; +//! use kernel::sync::{new_mutex, Mutex}; //! # use core::pin::Pin; //! #[pin_data] //! struct Foo { @@ -56,7 +56,7 @@ //! //! ```rust //! # #![allow(clippy::disallowed_names)] -//! # use kernel::{prelude::*, sync::Mutex, new_mutex}; +//! # use kernel::sync::{new_mutex, Mutex}; //! # use core::pin::Pin; //! # #[pin_data] //! # struct Foo { @@ -68,7 +68,7 @@ //! # a <- new_mutex!(42, "Foo::a"), //! # b: 24, //! # }); -//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo); +//! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo, GFP_KERNEL); //! ``` //! //! For more information see the [`pin_init!`] macro. @@ -79,15 +79,16 @@ //! above method only works for types where you can access the fields. //! //! ```rust -//! # use kernel::{new_mutex, sync::{Arc, Mutex}}; -//! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx")); +//! # use kernel::sync::{new_mutex, Arc, Mutex}; +//! let mtx: Result<Arc<Mutex<usize>>> = +//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL); //! ``` //! //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: //! //! ```rust //! # #![allow(clippy::disallowed_names)] -//! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; +//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init}; //! #[pin_data] //! struct DriverData { //! #[pin] @@ -99,7 +100,7 @@ //! fn new() -> impl PinInit<Self, Error> { //! try_pin_init!(Self { //! status <- new_mutex!(0, "DriverData::status"), -//! buffer: Box::init(kernel::init::zeroed())?, +//! buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?, //! }) //! } //! } @@ -121,7 +122,7 @@ //! //! ```rust //! # #![allow(unreachable_pub, clippy::disallowed_names)] -//! use kernel::{prelude::*, init, types::Opaque}; +//! use kernel::{init, types::Opaque}; //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; //! # mod bindings { //! # #![allow(non_camel_case_types)] @@ -210,13 +211,13 @@ //! [`pin_init!`]: crate::pin_init! use crate::{ + alloc::{box_ext::BoxExt, AllocError, Flags}, error::{self, Error}, sync::UniqueArc, types::{Opaque, ScopeGuard}, }; use alloc::boxed::Box; use core::{ - alloc::AllocError, cell::UnsafeCell, convert::Infallible, marker::PhantomData, @@ -305,9 +306,9 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::try_new(Bar { +/// b: Box::new(Bar { /// x: 64, -/// })?, +/// }, GFP_KERNEL)?, /// })); /// let foo = foo.unwrap(); /// pr_info!("a: {}", &*foo.a.lock()); @@ -331,9 +332,9 @@ macro_rules! stack_pin_init { /// /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { /// a <- new_mutex!(42), -/// b: Box::try_new(Bar { +/// b: Box::new(Bar { /// x: 64, -/// })?, +/// }, GFP_KERNEL)?, /// })); /// pr_info!("a: {}", &*foo.a.lock()); /// # Ok::<_, AllocError>(()) @@ -390,7 +391,7 @@ macro_rules! stack_try_pin_init { /// }, /// }); /// # initializer } -/// # Box::pin_init(demo()).unwrap(); +/// # Box::pin_init(demo(), GFP_KERNEL).unwrap(); /// ``` /// /// Arbitrary Rust expressions can be used to set the value of a variable. @@ -412,7 +413,7 @@ macro_rules! stack_try_pin_init { /// /// ```rust /// # #![allow(clippy::disallowed_names)] -/// # use kernel::{init, pin_init, prelude::*, init::*}; +/// # use kernel::{init, pin_init, init::*}; /// # use core::pin::Pin; /// # #[pin_data] /// # struct Foo { @@ -460,7 +461,7 @@ macro_rules! stack_try_pin_init { /// # }) /// # } /// # } -/// let foo = Box::pin_init(Foo::new()); +/// let foo = Box::pin_init(Foo::new(), GFP_KERNEL); /// ``` /// /// They can also easily embed it into their own `struct`s: @@ -600,7 +601,7 @@ macro_rules! pin_init { /// impl BigBuf { /// fn new() -> impl PinInit<Self, Error> { /// try_pin_init!(Self { -/// big: Box::init(init::zeroed())?, +/// big: Box::init(init::zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// ptr: core::ptr::null_mut(), /// }? Error) @@ -701,7 +702,7 @@ macro_rules! init { /// impl BigBuf { /// fn new() -> impl Init<Self, Error> { /// try_init!(Self { -/// big: Box::init(zeroed())?, +/// big: Box::init(zeroed(), GFP_KERNEL)?, /// small: [0; 1024 * 1024], /// }? Error) /// } @@ -751,10 +752,10 @@ macro_rules! try_init { /// /// # Safety /// -/// When implementing this type you will need to take great care. Also there are probably very few +/// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. /// -/// The [`PinInit::__pinned_init`] function +/// The [`PinInit::__pinned_init`] function: /// - returns `Ok(())` if it initialized every field of `slot`, /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: /// - `slot` can be deallocated without UB occurring, @@ -861,10 +862,10 @@ where /// /// # Safety /// -/// When implementing this type you will need to take great care. Also there are probably very few +/// When implementing this trait you will need to take great care. Also there are probably very few /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. /// -/// The [`Init::__init`] function +/// The [`Init::__init`] function: /// - returns `Ok(())` if it initialized every field of `slot`, /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: /// - `slot` can be deallocated without UB occurring, @@ -1013,7 +1014,7 @@ pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { /// /// ```rust /// use kernel::{error::Error, init::init_array_from_fn}; -/// let array: Box<[usize; 1_000]>= Box::init::<Error>(init_array_from_fn(|i| i)).unwrap(); +/// let array: Box<[usize; 1_000]> = Box::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` pub fn init_array_from_fn<I, const N: usize, T, E>( @@ -1027,7 +1028,7 @@ where // Counts the number of initialized elements and when dropped drops that many elements from // `slot`. let mut init_count = ScopeGuard::new_with_data(0, |i| { - // We now free every element that has been initialized before: + // We now free every element that has been initialized before. // SAFETY: The loop initialized exactly the values from 0..i and since we // return `Err` below, the caller will consider the memory at `slot` as // uninitialized. @@ -1056,8 +1057,8 @@ where /// /// ```rust /// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex}; -/// let array: Arc<[Mutex<usize>; 1_000]>= -/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i))).unwrap(); +/// let array: Arc<[Mutex<usize>; 1_000]> = +/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap(); /// assert_eq!(array.len(), 1_000); /// ``` pub fn pin_init_array_from_fn<I, const N: usize, T, E>( @@ -1071,7 +1072,7 @@ where // Counts the number of initialized elements and when dropped drops that many elements from // `slot`. let mut init_count = ScopeGuard::new_with_data(0, |i| { - // We now free every element that has been initialized before: + // We now free every element that has been initialized before. // SAFETY: The loop initialized exactly the values from 0..i and since we // return `Err` below, the caller will consider the memory at `slot` as // uninitialized. @@ -1115,7 +1116,7 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. - fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> where E: From<AllocError>; @@ -1123,7 +1124,7 @@ pub trait InPlaceInit<T>: Sized { /// type. /// /// If `T: !Unpin` it will not be able to move afterwards. - fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>> + fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Pin<Self>> where Error: From<E>, { @@ -1131,16 +1132,16 @@ pub trait InPlaceInit<T>: Sized { let init = unsafe { pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) }; - Self::try_pin_init(init) + Self::try_pin_init(init, flags) } /// Use the given initializer to in-place initialize a `T`. - fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> where E: From<AllocError>; /// Use the given initializer to in-place initialize a `T`. - fn init<E>(init: impl Init<T, E>) -> error::Result<Self> + fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> where Error: From<E>, { @@ -1148,17 +1149,17 @@ pub trait InPlaceInit<T>: Sized { let init = unsafe { init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) }; - Self::try_init(init) + Self::try_init(init, flags) } } impl<T> InPlaceInit<T> for Box<T> { #[inline] - fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> where E: From<AllocError>, { - let mut this = Box::try_new_uninit()?; + let mut this = <Box<_> as BoxExt<_>>::new_uninit(flags)?; let slot = this.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. @@ -1168,11 +1169,11 @@ impl<T> InPlaceInit<T> for Box<T> { } #[inline] - fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> where E: From<AllocError>, { - let mut this = Box::try_new_uninit()?; + let mut this = <Box<_> as BoxExt<_>>::new_uninit(flags)?; let slot = this.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. @@ -1184,11 +1185,11 @@ impl<T> InPlaceInit<T> for Box<T> { impl<T> InPlaceInit<T> for UniqueArc<T> { #[inline] - fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> + fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> where E: From<AllocError>, { - let mut this = UniqueArc::try_new_uninit()?; + let mut this = UniqueArc::new_uninit(flags)?; let slot = this.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid and will not be moved, because we pin it later. @@ -1198,11 +1199,11 @@ impl<T> InPlaceInit<T> for UniqueArc<T> { } #[inline] - fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> + fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> where E: From<AllocError>, { - let mut this = UniqueArc::try_new_uninit()?; + let mut this = UniqueArc::new_uninit(flags)?; let slot = this.as_mut_ptr(); // SAFETY: When init errors/panics, slot will get deallocated but not dropped, // slot is valid. @@ -1292,8 +1293,15 @@ impl_zeroable! { i8, i16, i32, i64, i128, isize, f32, f64, - // SAFETY: These are ZSTs, there is nothing to zero. - {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (), + // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list; + // creating an instance of an uninhabited type is immediate undefined behavior. For more on + // uninhabited/empty types, consult The Rustonomicon: + // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference + // also has information on undefined behavior: + // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>. + // + // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists. + {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (), // SAFETY: Type is allowed to take any value, including all zeros. {<T>} MaybeUninit<T>, diff --git a/rust/kernel/init/macros.rs b/rust/kernel/init/macros.rs index cb6e61b6c50b..02ecedc4ae7a 100644 --- a/rust/kernel/init/macros.rs +++ b/rust/kernel/init/macros.rs @@ -250,7 +250,7 @@ //! // error type is `Infallible`) we will need to drop this field if there //! // is an error later. This `DropGuard` will drop the field when it gets //! // dropped and has not yet been forgotten. -//! let t = unsafe { +//! let __t_guard = unsafe { //! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) //! }; //! // Expansion of `x: 0,`: @@ -261,14 +261,14 @@ //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; //! } //! // We again create a `DropGuard`. -//! let x = unsafe { +//! let __x_guard = unsafe { //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) //! }; //! // Since initialization has successfully completed, we can now forget //! // the guards. This is not `mem::forget`, since we only have //! // `&DropGuard`. -//! ::core::mem::forget(x); -//! ::core::mem::forget(t); +//! ::core::mem::forget(__x_guard); +//! ::core::mem::forget(__t_guard); //! // Here we use the type checker to ensure that every field has been //! // initialized exactly once, since this is `if false` it will never get //! // executed, but still type-checked. @@ -461,16 +461,16 @@ //! { //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; //! } -//! let a = unsafe { +//! let __a_guard = unsafe { //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) //! }; //! let init = Bar::new(36); //! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; -//! let b = unsafe { +//! let __b_guard = unsafe { //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) //! }; -//! ::core::mem::forget(b); -//! ::core::mem::forget(a); +//! ::core::mem::forget(__b_guard); +//! ::core::mem::forget(__a_guard); //! #[allow(unreachable_code, clippy::diverging_sub_expression)] //! let _ = || { //! unsafe { @@ -538,6 +538,7 @@ macro_rules! __pin_data { ), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @body({ $($fields:tt)* }), ) => { // We now use token munching to iterate through all of the fields. While doing this we @@ -560,6 +561,9 @@ macro_rules! __pin_data { @impl_generics($($impl_generics)*), // The 'ty generics', the generics that will need to be specified on the impl blocks. @ty_generics($($ty_generics)*), + // The 'decl generics', the generics that need to be specified on the struct + // definition. + @decl_generics($($decl_generics)*), // The where clause of any impl block and the declaration. @where($($($whr)*)?), // The remaining fields tokens that need to be processed. @@ -585,6 +589,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We found a PhantomPinned field, this should generally be pinned! @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), @@ -607,6 +612,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), @@ -623,6 +629,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration. @fields_munch($field:ident : $type:ty, $($rest:tt)*), @@ -640,6 +647,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)* $($accum)* $field: $type,), @@ -656,6 +664,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration. @fields_munch($field:ident : $type:ty, $($rest:tt)*), @@ -673,6 +682,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)*), @@ -689,6 +699,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We found the `#[pin]` attr. @fields_munch(#[pin] $($rest:tt)*), @@ -705,6 +716,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), // We do not include `#[pin]` in the list of attributes, since it is not actually an @@ -724,6 +736,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the field declaration with visibility, for simplicity we only munch the // visibility and put it into `$accum`. @@ -741,6 +754,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($field $($rest)*), @pinned($($pinned)*), @@ -757,6 +771,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // Some other attribute, just put it into `$accum`. @fields_munch(#[$($attr:tt)*] $($rest:tt)*), @@ -773,6 +788,7 @@ macro_rules! __pin_data { @name($name), @impl_generics($($impl_generics)*), @ty_generics($($ty_generics)*), + @decl_generics($($decl_generics)*), @where($($whr)*), @fields_munch($($rest)*), @pinned($($pinned)*), @@ -789,6 +805,7 @@ macro_rules! __pin_data { @name($name:ident), @impl_generics($($impl_generics:tt)*), @ty_generics($($ty_generics:tt)*), + @decl_generics($($decl_generics:tt)*), @where($($whr:tt)*), // We reached the end of the fields, plus an optional additional comma, since we added one // before and the user is also allowed to put a trailing comma. @@ -802,7 +819,7 @@ macro_rules! __pin_data { ) => { // Declare the struct with all fields in the correct order. $($struct_attrs)* - $vis struct $name <$($impl_generics)*> + $vis struct $name <$($decl_generics)*> where $($whr)* { $($fields)* @@ -1192,14 +1209,14 @@ macro_rules! __init_internal { // We use `paste!` to create new hygiene for `$field`. ::kernel::macros::paste! { // SAFETY: We forget the guard later when initialization has succeeded. - let [<$field>] = unsafe { + let [< __ $field _guard >] = unsafe { $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::__init_internal!(init_slot($use_data): @data($data), @slot($slot), - @guards([<$field>], $($guards,)*), + @guards([< __ $field _guard >], $($guards,)*), @munch_fields($($rest)*), ); } @@ -1223,14 +1240,14 @@ macro_rules! __init_internal { // We use `paste!` to create new hygiene for `$field`. ::kernel::macros::paste! { // SAFETY: We forget the guard later when initialization has succeeded. - let [<$field>] = unsafe { + let [< __ $field _guard >] = unsafe { $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::__init_internal!(init_slot(): @data($data), @slot($slot), - @guards([<$field>], $($guards,)*), + @guards([< __ $field _guard >], $($guards,)*), @munch_fields($($rest)*), ); } @@ -1255,14 +1272,14 @@ macro_rules! __init_internal { // We use `paste!` to create new hygiene for `$field`. ::kernel::macros::paste! { // SAFETY: We forget the guard later when initialization has succeeded. - let [<$field>] = unsafe { + let [< __ $field _guard >] = unsafe { $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) }; $crate::__init_internal!(init_slot($($use_data)?): @data($data), @slot($slot), - @guards([<$field>], $($guards,)*), + @guards([< __ $field _guard >], $($guards,)*), @munch_fields($($rest)*), ); } diff --git a/rust/kernel/ioctl.rs b/rust/kernel/ioctl.rs index c49e1a8d3fd0..cfa7d080b531 100644 --- a/rust/kernel/ioctl.rs +++ b/rust/kernel/ioctl.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 -//! ioctl() number definitions +//! `ioctl()` number definitions. //! -//! C header: [`include/asm-generic/ioctl.h`](../../../../include/asm-generic/ioctl.h) +//! C header: [`include/asm-generic/ioctl.h`](srctree/include/asm-generic/ioctl.h) #![allow(non_snake_case)] @@ -28,13 +28,13 @@ pub const fn _IO(ty: u32, nr: u32) -> u32 { _IOC(uapi::_IOC_NONE, ty, nr, 0) } -/// Build an ioctl number for an read-only ioctl. +/// Build an ioctl number for a read-only ioctl. #[inline(always)] pub const fn _IOR<T>(ty: u32, nr: u32) -> u32 { _IOC(uapi::_IOC_READ, ty, nr, core::mem::size_of::<T>()) } -/// Build an ioctl number for an write-only ioctl. +/// Build an ioctl number for a write-only ioctl. #[inline(always)] pub const fn _IOW<T>(ty: u32, nr: u32) -> u32 { _IOC(uapi::_IOC_WRITE, ty, nr, core::mem::size_of::<T>()) diff --git a/rust/kernel/kunit.rs b/rust/kernel/kunit.rs index 722655b2d62d..0ba77276ae7e 100644 --- a/rust/kernel/kunit.rs +++ b/rust/kernel/kunit.rs @@ -2,7 +2,7 @@ //! KUnit-based macros for Rust unit tests. //! -//! C header: [`include/kunit/test.h`](../../../../../include/kunit/test.h) +//! C header: [`include/kunit/test.h`](srctree/include/kunit/test.h) //! //! Reference: <https://docs.kernel.org/dev-tools/kunit/index.html> diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index e6aff80b521f..fbd91a48ff8b 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -12,12 +12,9 @@ //! do so first instead of bypassing this crate. #![no_std] -#![feature(allocator_api)] #![feature(coerce_unsized)] #![feature(dispatch_from_dyn)] #![feature(new_uninit)] -#![feature(offset_of)] -#![feature(ptr_metadata)] #![feature(receiver_trait)] #![feature(unsize)] @@ -29,15 +26,15 @@ compile_error!("Missing kernel configuration for conditional compilation"); // Allow proc-macros to refer to `::kernel` inside the `kernel` crate (this crate). extern crate self as kernel; -#[cfg(not(test))] -#[cfg(not(testlib))] -mod allocator; +pub mod alloc; mod build_assert; pub mod error; pub mod init; pub mod ioctl; #[cfg(CONFIG_KUNIT)] pub mod kunit; +#[cfg(CONFIG_NET)] +pub mod net; pub mod prelude; pub mod print; mod static_assert; @@ -46,6 +43,7 @@ pub mod std_vendor; pub mod str; pub mod sync; pub mod task; +pub mod time; pub mod types; pub mod workqueue; @@ -63,7 +61,7 @@ const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; /// The top level entrypoint to implementing a kernel module. /// /// For any teardown or cleanup operations, your type may implement [`Drop`]. -pub trait Module: Sized + Sync { +pub trait Module: Sized + Sync + Send { /// Called at module initialization time. /// /// Use this method to perform whatever setup or registration your module @@ -75,7 +73,7 @@ pub trait Module: Sized + Sync { /// Equivalent to `THIS_MODULE` in the C API. /// -/// C header: `include/linux/export.h` +/// C header: [`include/linux/export.h`](srctree/include/linux/export.h) pub struct ThisModule(*mut bindings::module); // SAFETY: `THIS_MODULE` may be used from all threads within a module. @@ -90,6 +88,13 @@ impl ThisModule { pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule { ThisModule(ptr) } + + /// Access the raw pointer for this module. + /// + /// It is up to the user to use it correctly. + pub const fn as_ptr(&self) -> *mut bindings::module { + self.0 + } } #[cfg(not(any(testlib, test)))] @@ -99,3 +104,35 @@ fn panic(info: &core::panic::PanicInfo<'_>) -> ! { // SAFETY: FFI call. unsafe { bindings::BUG() }; } + +/// Produces a pointer to an object from a pointer to one of its fields. +/// +/// # Safety +/// +/// The pointer passed to this macro, and the pointer returned by this macro, must both be in +/// bounds of the same allocation. +/// +/// # Examples +/// +/// ``` +/// # use kernel::container_of; +/// struct Test { +/// a: u64, +/// b: u32, +/// } +/// +/// let test = Test { a: 10, b: 20 }; +/// let b_ptr = &test.b; +/// // SAFETY: The pointer points at the `b` field of a `Test`, so the resulting pointer will be +/// // in-bounds of the same allocation as `b_ptr`. +/// let test_alias = unsafe { container_of!(b_ptr, Test, b) }; +/// assert!(core::ptr::eq(&test, test_alias)); +/// ``` +#[macro_export] +macro_rules! container_of { + ($ptr:expr, $type:ty, $($f:tt)*) => {{ + let ptr = $ptr as *const _ as *const u8; + let offset: usize = ::core::mem::offset_of!($type, $($f)*); + ptr.sub(offset) as *const $type + }} +} diff --git a/rust/kernel/net.rs b/rust/kernel/net.rs new file mode 100644 index 000000000000..fe415cb369d3 --- /dev/null +++ b/rust/kernel/net.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Networking. + +#[cfg(CONFIG_RUST_PHYLIB_ABSTRACTIONS)] +pub mod phy; diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs new file mode 100644 index 000000000000..fd40b703d224 --- /dev/null +++ b/rust/kernel/net/phy.rs @@ -0,0 +1,905 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Copyright (C) 2023 FUJITA Tomonori <fujita.tomonori@gmail.com> + +//! Network PHY device. +//! +//! C headers: [`include/linux/phy.h`](srctree/include/linux/phy.h). + +use crate::{error::*, prelude::*, types::Opaque}; + +use core::marker::PhantomData; + +/// PHY state machine states. +/// +/// Corresponds to the kernel's [`enum phy_state`]. +/// +/// Some of PHY drivers access to the state of PHY's software state machine. +/// +/// [`enum phy_state`]: srctree/include/linux/phy.h +#[derive(PartialEq, Eq)] +pub enum DeviceState { + /// PHY device and driver are not ready for anything. + Down, + /// PHY is ready to send and receive packets. + Ready, + /// PHY is up, but no polling or interrupts are done. + Halted, + /// PHY is up, but is in an error state. + Error, + /// PHY and attached device are ready to do work. + Up, + /// PHY is currently running. + Running, + /// PHY is up, but not currently plugged in. + NoLink, + /// PHY is performing a cable test. + CableTest, +} + +/// A mode of Ethernet communication. +/// +/// PHY drivers get duplex information from hardware and update the current state. +pub enum DuplexMode { + /// PHY is in full-duplex mode. + Full, + /// PHY is in half-duplex mode. + Half, + /// PHY is in unknown duplex mode. + Unknown, +} + +/// An instance of a PHY device. +/// +/// Wraps the kernel's [`struct phy_device`]. +/// +/// A [`Device`] instance is created when a callback in [`Driver`] is executed. A PHY driver +/// executes [`Driver`]'s methods during the callback. +/// +/// # Invariants +/// +/// Referencing a `phy_device` using this struct asserts that you are in +/// a context where all methods defined on this struct are safe to call. +/// +/// [`struct phy_device`]: srctree/include/linux/phy.h +// During the calls to most functions in [`Driver`], the C side (`PHYLIB`) holds a lock that is +// unique for every instance of [`Device`]. `PHYLIB` uses a different serialization technique for +// [`Driver::resume`] and [`Driver::suspend`]: `PHYLIB` updates `phy_device`'s state with +// the lock held, thus guaranteeing that [`Driver::resume`] has exclusive access to the instance. +// [`Driver::resume`] and [`Driver::suspend`] also are called where only one thread can access +// to the instance. +#[repr(transparent)] +pub struct Device(Opaque<bindings::phy_device>); + +impl Device { + /// Creates a new [`Device`] instance from a raw pointer. + /// + /// # Safety + /// + /// For the duration of 'a, the pointer must point at a valid `phy_device`, + /// and the caller must be in a context where all methods defined on this struct + /// are safe to call. + unsafe fn from_raw<'a>(ptr: *mut bindings::phy_device) -> &'a mut Self { + // CAST: `Self` is a `repr(transparent)` wrapper around `bindings::phy_device`. + let ptr = ptr.cast::<Self>(); + // SAFETY: by the function requirements the pointer is valid and we have unique access for + // the duration of `'a`. + unsafe { &mut *ptr } + } + + /// Gets the id of the PHY. + pub fn phy_id(&self) -> u32 { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).phy_id } + } + + /// Gets the state of PHY state machine states. + pub fn state(&self) -> DeviceState { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let state = unsafe { (*phydev).state }; + // TODO: this conversion code will be replaced with automatically generated code by bindgen + // when it becomes possible. + match state { + bindings::phy_state_PHY_DOWN => DeviceState::Down, + bindings::phy_state_PHY_READY => DeviceState::Ready, + bindings::phy_state_PHY_HALTED => DeviceState::Halted, + bindings::phy_state_PHY_ERROR => DeviceState::Error, + bindings::phy_state_PHY_UP => DeviceState::Up, + bindings::phy_state_PHY_RUNNING => DeviceState::Running, + bindings::phy_state_PHY_NOLINK => DeviceState::NoLink, + bindings::phy_state_PHY_CABLETEST => DeviceState::CableTest, + _ => DeviceState::Error, + } + } + + /// Gets the current link state. + /// + /// It returns true if the link is up. + pub fn is_link_up(&self) -> bool { + const LINK_IS_UP: u64 = 1; + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(14, 1) == LINK_IS_UP + } + + /// Gets the current auto-negotiation configuration. + /// + /// It returns true if auto-negotiation is enabled. + pub fn is_autoneg_enabled(&self) -> bool { + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(13, 1) == bindings::AUTONEG_ENABLE as u64 + } + + /// Gets the current auto-negotiation state. + /// + /// It returns true if auto-negotiation is completed. + pub fn is_autoneg_completed(&self) -> bool { + const AUTONEG_COMPLETED: u64 = 1; + // TODO: the code to access to the bit field will be replaced with automatically + // generated code by bindgen when it becomes possible. + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + let bit_field = unsafe { &(*self.0.get())._bitfield_1 }; + bit_field.get(15, 1) == AUTONEG_COMPLETED + } + + /// Sets the speed of the PHY. + pub fn set_speed(&mut self, speed: u32) { + let phydev = self.0.get(); + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).speed = speed as i32 }; + } + + /// Sets duplex mode. + pub fn set_duplex(&mut self, mode: DuplexMode) { + let phydev = self.0.get(); + let v = match mode { + DuplexMode::Full => bindings::DUPLEX_FULL as i32, + DuplexMode::Half => bindings::DUPLEX_HALF as i32, + DuplexMode::Unknown => bindings::DUPLEX_UNKNOWN as i32, + }; + // SAFETY: The struct invariant ensures that we may access + // this field without additional synchronization. + unsafe { (*phydev).duplex = v }; + } + + /// Reads a given C22 PHY register. + // This function reads a hardware register and updates the stats so takes `&mut self`. + pub fn read(&mut self, regnum: u16) -> Result<u16> { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call, open code of `phy_read()` with a valid `phy_device` pointer + // `phydev`. + let ret = unsafe { + bindings::mdiobus_read((*phydev).mdio.bus, (*phydev).mdio.addr, regnum.into()) + }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Writes a given C22 PHY register. + pub fn write(&mut self, regnum: u16, val: u16) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call, open code of `phy_write()` with a valid `phy_device` pointer + // `phydev`. + to_result(unsafe { + bindings::mdiobus_write((*phydev).mdio.bus, (*phydev).mdio.addr, regnum.into(), val) + }) + } + + /// Reads a paged register. + pub fn read_paged(&mut self, page: u16, regnum: u16) -> Result<u16> { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::phy_read_paged(phydev, page.into(), regnum.into()) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Resolves the advertisements into PHY settings. + pub fn resolve_aneg_linkmode(&mut self) { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + unsafe { bindings::phy_resolve_aneg_linkmode(phydev) }; + } + + /// Executes software reset the PHY via `BMCR_RESET` bit. + pub fn genphy_soft_reset(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_soft_reset(phydev) }) + } + + /// Initializes the PHY. + pub fn init_hw(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::phy_init_hw(phydev) }) + } + + /// Starts auto-negotiation. + pub fn start_aneg(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::_phy_start_aneg(phydev) }) + } + + /// Resumes the PHY via `BMCR_PDOWN` bit. + pub fn genphy_resume(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_resume(phydev) }) + } + + /// Suspends the PHY via `BMCR_PDOWN` bit. + pub fn genphy_suspend(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_suspend(phydev) }) + } + + /// Checks the link status and updates current link state. + pub fn genphy_read_status(&mut self) -> Result<u16> { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + let ret = unsafe { bindings::genphy_read_status(phydev) }; + if ret < 0 { + Err(Error::from_errno(ret)) + } else { + Ok(ret as u16) + } + } + + /// Updates the link status. + pub fn genphy_update_link(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_update_link(phydev) }) + } + + /// Reads link partner ability. + pub fn genphy_read_lpa(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_lpa(phydev) }) + } + + /// Reads PHY abilities. + pub fn genphy_read_abilities(&mut self) -> Result { + let phydev = self.0.get(); + // SAFETY: `phydev` is pointing to a valid object by the type invariant of `Self`. + // So it's just an FFI call. + to_result(unsafe { bindings::genphy_read_abilities(phydev) }) + } +} + +/// Defines certain other features this PHY supports (like interrupts). +/// +/// These flag values are used in [`Driver::FLAGS`]. +pub mod flags { + /// PHY is internal. + pub const IS_INTERNAL: u32 = bindings::PHY_IS_INTERNAL; + /// PHY needs to be reset after the refclk is enabled. + pub const RST_AFTER_CLK_EN: u32 = bindings::PHY_RST_AFTER_CLK_EN; + /// Polling is used to detect PHY status changes. + pub const POLL_CABLE_TEST: u32 = bindings::PHY_POLL_CABLE_TEST; + /// Don't suspend. + pub const ALWAYS_CALL_SUSPEND: u32 = bindings::PHY_ALWAYS_CALL_SUSPEND; +} + +/// An adapter for the registration of a PHY driver. +struct Adapter<T: Driver> { + _p: PhantomData<T>, +} + +impl<T: Driver> Adapter<T> { + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn soft_reset_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::soft_reset(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn get_features_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::get_features(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn suspend_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::suspend(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn resume_callback(phydev: *mut bindings::phy_device) -> core::ffi::c_int { + from_result(|| { + // SAFETY: The C core code ensures that the accessors on + // `Device` are okay to call even though `phy_device->lock` + // might not be held. + let dev = unsafe { Device::from_raw(phydev) }; + T::resume(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn config_aneg_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::config_aneg(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_status_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::read_status(dev)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn match_phy_device_callback( + phydev: *mut bindings::phy_device, + ) -> core::ffi::c_int { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::match_phy_device(dev) as i32 + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn read_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + // CAST: the C side verifies devnum < 32. + let ret = T::read_mmd(dev, devnum as u8, regnum)?; + Ok(ret.into()) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn write_mmd_callback( + phydev: *mut bindings::phy_device, + devnum: i32, + regnum: u16, + val: u16, + ) -> i32 { + from_result(|| { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::write_mmd(dev, devnum as u8, regnum, val)?; + Ok(0) + }) + } + + /// # Safety + /// + /// `phydev` must be passed by the corresponding callback in `phy_driver`. + unsafe extern "C" fn link_change_notify_callback(phydev: *mut bindings::phy_device) { + // SAFETY: This callback is called only in contexts + // where we hold `phy_device->lock`, so the accessors on + // `Device` are okay to call. + let dev = unsafe { Device::from_raw(phydev) }; + T::link_change_notify(dev); + } +} + +/// Driver structure for a particular PHY type. +/// +/// Wraps the kernel's [`struct phy_driver`]. +/// This is used to register a driver for a particular PHY type with the kernel. +/// +/// # Invariants +/// +/// `self.0` is always in a valid state. +/// +/// [`struct phy_driver`]: srctree/include/linux/phy.h +#[repr(transparent)] +pub struct DriverVTable(Opaque<bindings::phy_driver>); + +// SAFETY: `DriverVTable` doesn't expose any &self method to access internal data, so it's safe to +// share `&DriverVTable` across execution context boundries. +unsafe impl Sync for DriverVTable {} + +/// Creates a [`DriverVTable`] instance from [`Driver`]. +/// +/// This is used by [`module_phy_driver`] macro to create a static array of `phy_driver`. +/// +/// [`module_phy_driver`]: crate::module_phy_driver +pub const fn create_phy_driver<T: Driver>() -> DriverVTable { + // INVARIANT: All the fields of `struct phy_driver` are initialized properly. + DriverVTable(Opaque::new(bindings::phy_driver { + name: T::NAME.as_char_ptr().cast_mut(), + flags: T::FLAGS, + phy_id: T::PHY_DEVICE_ID.id, + phy_id_mask: T::PHY_DEVICE_ID.mask_as_int(), + soft_reset: if T::HAS_SOFT_RESET { + Some(Adapter::<T>::soft_reset_callback) + } else { + None + }, + get_features: if T::HAS_GET_FEATURES { + Some(Adapter::<T>::get_features_callback) + } else { + None + }, + match_phy_device: if T::HAS_MATCH_PHY_DEVICE { + Some(Adapter::<T>::match_phy_device_callback) + } else { + None + }, + suspend: if T::HAS_SUSPEND { + Some(Adapter::<T>::suspend_callback) + } else { + None + }, + resume: if T::HAS_RESUME { + Some(Adapter::<T>::resume_callback) + } else { + None + }, + config_aneg: if T::HAS_CONFIG_ANEG { + Some(Adapter::<T>::config_aneg_callback) + } else { + None + }, + read_status: if T::HAS_READ_STATUS { + Some(Adapter::<T>::read_status_callback) + } else { + None + }, + read_mmd: if T::HAS_READ_MMD { + Some(Adapter::<T>::read_mmd_callback) + } else { + None + }, + write_mmd: if T::HAS_WRITE_MMD { + Some(Adapter::<T>::write_mmd_callback) + } else { + None + }, + link_change_notify: if T::HAS_LINK_CHANGE_NOTIFY { + Some(Adapter::<T>::link_change_notify_callback) + } else { + None + }, + // SAFETY: The rest is zeroed out to initialize `struct phy_driver`, + // sets `Option<&F>` to be `None`. + ..unsafe { core::mem::MaybeUninit::<bindings::phy_driver>::zeroed().assume_init() } + })) +} + +/// Driver implementation for a particular PHY type. +/// +/// This trait is used to create a [`DriverVTable`]. +#[vtable] +pub trait Driver { + /// Defines certain other features this PHY supports. + /// It is a combination of the flags in the [`flags`] module. + const FLAGS: u32 = 0; + + /// The friendly name of this PHY type. + const NAME: &'static CStr; + + /// This driver only works for PHYs with IDs which match this field. + /// The default id and mask are zero. + const PHY_DEVICE_ID: DeviceId = DeviceId::new_with_custom_mask(0, 0); + + /// Issues a PHY software reset. + fn soft_reset(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Probes the hardware to determine what abilities it has. + fn get_features(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Returns true if this is a suitable driver for the given phydev. + /// If not implemented, matching is based on [`Driver::PHY_DEVICE_ID`]. + fn match_phy_device(_dev: &Device) -> bool { + false + } + + /// Configures the advertisement and resets auto-negotiation + /// if auto-negotiation is enabled. + fn config_aneg(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Determines the negotiated speed and duplex. + fn read_status(_dev: &mut Device) -> Result<u16> { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Suspends the hardware, saving state if needed. + fn suspend(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Resumes the hardware, restoring state if needed. + fn resume(_dev: &mut Device) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Overrides the default MMD read function for reading a MMD register. + fn read_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16) -> Result<u16> { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Overrides the default MMD write function for writing a MMD register. + fn write_mmd(_dev: &mut Device, _devnum: u8, _regnum: u16, _val: u16) -> Result { + kernel::build_error(VTABLE_DEFAULT_ERROR) + } + + /// Callback for notification of link change. + fn link_change_notify(_dev: &mut Device) {} +} + +/// Registration structure for PHY drivers. +/// +/// Registers [`DriverVTable`] instances with the kernel. They will be unregistered when dropped. +/// +/// # Invariants +/// +/// The `drivers` slice are currently registered to the kernel via `phy_drivers_register`. +pub struct Registration { + drivers: Pin<&'static mut [DriverVTable]>, +} + +// SAFETY: The only action allowed in a `Registration` instance is dropping it, which is safe to do +// from any thread because `phy_drivers_unregister` can be called from any thread context. +unsafe impl Send for Registration {} + +impl Registration { + /// Registers a PHY driver. + pub fn register( + module: &'static crate::ThisModule, + drivers: Pin<&'static mut [DriverVTable]>, + ) -> Result<Self> { + if drivers.is_empty() { + return Err(code::EINVAL); + } + // SAFETY: The type invariants of [`DriverVTable`] ensure that all elements of + // the `drivers` slice are initialized properly. `drivers` will not be moved. + // So it's just an FFI call. + to_result(unsafe { + bindings::phy_drivers_register(drivers[0].0.get(), drivers.len().try_into()?, module.0) + })?; + // INVARIANT: The `drivers` slice is successfully registered to the kernel via `phy_drivers_register`. + Ok(Registration { drivers }) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that `self.drivers` is valid. + // So it's just an FFI call. + unsafe { + bindings::phy_drivers_unregister(self.drivers[0].0.get(), self.drivers.len() as i32) + }; + } +} + +/// An identifier for PHY devices on an MDIO/MII bus. +/// +/// Represents the kernel's `struct mdio_device_id`. This is used to find an appropriate +/// PHY driver. +pub struct DeviceId { + id: u32, + mask: DeviceMask, +} + +impl DeviceId { + /// Creates a new instance with the exact match mask. + pub const fn new_with_exact_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Exact, + } + } + + /// Creates a new instance with the model match mask. + pub const fn new_with_model_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Model, + } + } + + /// Creates a new instance with the vendor match mask. + pub const fn new_with_vendor_mask(id: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Vendor, + } + } + + /// Creates a new instance with a custom match mask. + pub const fn new_with_custom_mask(id: u32, mask: u32) -> Self { + DeviceId { + id, + mask: DeviceMask::Custom(mask), + } + } + + /// Creates a new instance from [`Driver`]. + pub const fn new_with_driver<T: Driver>() -> Self { + T::PHY_DEVICE_ID + } + + /// Get a `mask` as u32. + pub const fn mask_as_int(&self) -> u32 { + self.mask.as_int() + } + + // macro use only + #[doc(hidden)] + pub const fn mdio_device_id(&self) -> bindings::mdio_device_id { + bindings::mdio_device_id { + phy_id: self.id, + phy_id_mask: self.mask.as_int(), + } + } +} + +enum DeviceMask { + Exact, + Model, + Vendor, + Custom(u32), +} + +impl DeviceMask { + const MASK_EXACT: u32 = !0; + const MASK_MODEL: u32 = !0 << 4; + const MASK_VENDOR: u32 = !0 << 10; + + const fn as_int(&self) -> u32 { + match self { + DeviceMask::Exact => Self::MASK_EXACT, + DeviceMask::Model => Self::MASK_MODEL, + DeviceMask::Vendor => Self::MASK_VENDOR, + DeviceMask::Custom(mask) => *mask, + } + } +} + +/// Declares a kernel module for PHYs drivers. +/// +/// This creates a static array of kernel's `struct phy_driver` and registers it. +/// This also corresponds to the kernel's `MODULE_DEVICE_TABLE` macro, which embeds the information +/// for module loading into the module binary file. Every driver needs an entry in `device_table`. +/// +/// # Examples +/// +/// ``` +/// # mod module_phy_driver_sample { +/// use kernel::c_str; +/// use kernel::net::phy::{self, DeviceId}; +/// use kernel::prelude::*; +/// +/// kernel::module_phy_driver! { +/// drivers: [PhySample], +/// device_table: [ +/// DeviceId::new_with_driver::<PhySample>() +/// ], +/// name: "rust_sample_phy", +/// author: "Rust for Linux Contributors", +/// description: "Rust sample PHYs driver", +/// license: "GPL", +/// } +/// +/// struct PhySample; +/// +/// #[vtable] +/// impl phy::Driver for PhySample { +/// const NAME: &'static CStr = c_str!("PhySample"); +/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); +/// } +/// # } +/// ``` +/// +/// This expands to the following code: +/// +/// ```ignore +/// use kernel::c_str; +/// use kernel::net::phy::{self, DeviceId}; +/// use kernel::prelude::*; +/// +/// struct Module { +/// _reg: ::kernel::net::phy::Registration, +/// } +/// +/// module! { +/// type: Module, +/// name: "rust_sample_phy", +/// author: "Rust for Linux Contributors", +/// description: "Rust sample PHYs driver", +/// license: "GPL", +/// } +/// +/// struct PhySample; +/// +/// #[vtable] +/// impl phy::Driver for PhySample { +/// const NAME: &'static CStr = c_str!("PhySample"); +/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); +/// } +/// +/// const _: () = { +/// static mut DRIVERS: [::kernel::net::phy::DriverVTable; 1] = +/// [::kernel::net::phy::create_phy_driver::<PhySample>()]; +/// +/// impl ::kernel::Module for Module { +/// fn init(module: &'static ThisModule) -> Result<Self> { +/// let drivers = unsafe { &mut DRIVERS }; +/// let mut reg = ::kernel::net::phy::Registration::register( +/// module, +/// ::core::pin::Pin::static_mut(drivers), +/// )?; +/// Ok(Module { _reg: reg }) +/// } +/// } +/// }; +/// +/// #[cfg(MODULE)] +/// #[no_mangle] +/// static __mod_mdio__phydev_device_table: [::kernel::bindings::mdio_device_id; 2] = [ +/// ::kernel::bindings::mdio_device_id { +/// phy_id: 0x00000001, +/// phy_id_mask: 0xffffffff, +/// }, +/// ::kernel::bindings::mdio_device_id { +/// phy_id: 0, +/// phy_id_mask: 0, +/// }, +/// ]; +/// ``` +#[macro_export] +macro_rules! module_phy_driver { + (@replace_expr $_t:tt $sub:expr) => {$sub}; + + (@count_devices $($x:expr),*) => { + 0usize $(+ $crate::module_phy_driver!(@replace_expr $x 1usize))* + }; + + (@device_table [$($dev:expr),+]) => { + // SAFETY: C will not read off the end of this constant since the last element is zero. + #[cfg(MODULE)] + #[no_mangle] + static __mod_mdio__phydev_device_table: [$crate::bindings::mdio_device_id; + $crate::module_phy_driver!(@count_devices $($dev),+) + 1] = [ + $($dev.mdio_device_id()),+, + $crate::bindings::mdio_device_id { + phy_id: 0, + phy_id_mask: 0 + } + ]; + }; + + (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { + struct Module { + _reg: $crate::net::phy::Registration, + } + + $crate::prelude::module! { + type: Module, + $($f)* + } + + const _: () = { + static mut DRIVERS: [$crate::net::phy::DriverVTable; + $crate::module_phy_driver!(@count_devices $($driver),+)] = + [$($crate::net::phy::create_phy_driver::<$driver>()),+]; + + impl $crate::Module for Module { + fn init(module: &'static ThisModule) -> Result<Self> { + // SAFETY: The anonymous constant guarantees that nobody else can access + // the `DRIVERS` static. The array is used only in the C side. + let drivers = unsafe { &mut DRIVERS }; + let mut reg = $crate::net::phy::Registration::register( + module, + ::core::pin::Pin::static_mut(drivers), + )?; + Ok(Module { _reg: reg }) + } + } + }; + + $crate::module_phy_driver!(@device_table [$($dev),+]); + } +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs index ae21600970b3..b37a0b3180fb 100644 --- a/rust/kernel/prelude.rs +++ b/rust/kernel/prelude.rs @@ -14,6 +14,8 @@ #[doc(no_inline)] pub use core::pin::Pin; +pub use crate::alloc::{box_ext::BoxExt, flags::*, vec_ext::VecExt}; + #[doc(no_inline)] pub use alloc::{boxed::Box, vec::Vec}; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs index f48926e3e9fe..a78aa3514a0a 100644 --- a/rust/kernel/print.rs +++ b/rust/kernel/print.rs @@ -2,7 +2,7 @@ //! Printing facilities. //! -//! C header: [`include/linux/printk.h`](../../../../include/linux/printk.h) +//! C header: [`include/linux/printk.h`](srctree/include/linux/printk.h) //! //! Reference: <https://www.kernel.org/doc/html/latest/core-api/printk-basics.html> @@ -13,9 +13,6 @@ use core::{ use crate::str::RawFormatter; -#[cfg(CONFIG_PRINTK)] -use crate::bindings; - // Called from `vsprintf` with format specifier `%pA`. #[no_mangle] unsafe extern "C" fn rust_fmt_argument( @@ -35,8 +32,6 @@ unsafe extern "C" fn rust_fmt_argument( /// Public but hidden since it should only be used from public macros. #[doc(hidden)] pub mod format_strings { - use crate::bindings; - /// The length we copy from the `KERN_*` kernel prefixes. const LENGTH_PREFIX: usize = 2; @@ -48,7 +43,7 @@ pub mod format_strings { /// The format string is always the same for a given level, i.e. for a /// given `prefix`, which are the kernel's `KERN_*` constants. /// - /// [`_printk`]: ../../../../include/linux/printk.h + /// [`_printk`]: srctree/include/linux/printk.h const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] { // Ensure the `KERN_*` macros are what we expect. assert!(prefix[0] == b'\x01'); @@ -97,7 +92,7 @@ pub mod format_strings { /// The format string must be one of the ones in [`format_strings`], and /// the module name must be null-terminated. /// -/// [`_printk`]: ../../../../include/linux/_printk.h +/// [`_printk`]: srctree/include/linux/_printk.h #[doc(hidden)] #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] pub unsafe fn call_printk( @@ -120,7 +115,7 @@ pub unsafe fn call_printk( /// /// Public but hidden since it should only be used from public macros. /// -/// [`_printk`]: ../../../../include/linux/printk.h +/// [`_printk`]: srctree/include/linux/printk.h #[doc(hidden)] #[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] pub fn call_printk_cont(args: fmt::Arguments<'_>) { diff --git a/rust/kernel/std_vendor.rs b/rust/kernel/std_vendor.rs index 388d6a5147a2..39679a960c1a 100644 --- a/rust/kernel/std_vendor.rs +++ b/rust/kernel/std_vendor.rs @@ -146,15 +146,16 @@ macro_rules! dbg { // `$val` expression could be a block (`{ .. }`), in which case the `pr_info!` // will be malformed. () => { - $crate::pr_info!("[{}:{}]\n", ::core::file!(), ::core::line!()) + $crate::pr_info!("[{}:{}:{}]\n", ::core::file!(), ::core::line!(), ::core::column!()) }; ($val:expr $(,)?) => { // Use of `match` here is intentional because it affects the lifetimes // of temporaries - https://stackoverflow.com/a/48732525/1063961 match $val { tmp => { - $crate::pr_info!("[{}:{}] {} = {:#?}\n", - ::core::file!(), ::core::line!(), ::core::stringify!($val), &tmp); + $crate::pr_info!("[{}:{}:{}] {} = {:#?}\n", + ::core::file!(), ::core::line!(), ::core::column!(), + ::core::stringify!($val), &tmp); tmp } } diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs index c41607b2e4fe..bb8d4f41475b 100644 --- a/rust/kernel/str.rs +++ b/rust/kernel/str.rs @@ -2,20 +2,110 @@ //! String representations. -use alloc::alloc::AllocError; +use crate::alloc::{flags::*, vec_ext::VecExt, AllocError}; use alloc::vec::Vec; use core::fmt::{self, Write}; -use core::ops::{self, Deref, Index}; +use core::ops::{self, Deref, DerefMut, Index}; -use crate::{ - bindings, - error::{code::*, Error}, -}; +use crate::error::{code::*, Error}; /// Byte string without UTF-8 validity guarantee. -/// -/// `BStr` is simply an alias to `[u8]`, but has a more evident semantical meaning. -pub type BStr = [u8]; +#[repr(transparent)] +pub struct BStr([u8]); + +impl BStr { + /// Returns the length of this string. + #[inline] + pub const fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the string is empty. + #[inline] + pub const fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Creates a [`BStr`] from a `[u8]`. + #[inline] + pub const fn from_bytes(bytes: &[u8]) -> &Self { + // SAFETY: `BStr` is transparent to `[u8]`. + unsafe { &*(bytes as *const [u8] as *const BStr) } + } +} + +impl fmt::Display for BStr { + /// Formats printable ASCII characters, escaping the rest. + /// + /// ``` + /// # use kernel::{fmt, b_str, str::{BStr, CString}}; + /// let ascii = b_str!("Hello, BStr!"); + /// let s = CString::try_from_fmt(fmt!("{}", ascii)).unwrap(); + /// assert_eq!(s.as_bytes(), "Hello, BStr!".as_bytes()); + /// + /// let non_ascii = b_str!("🦀"); + /// let s = CString::try_from_fmt(fmt!("{}", non_ascii)).unwrap(); + /// assert_eq!(s.as_bytes(), "\\xf0\\x9f\\xa6\\x80".as_bytes()); + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for &b in &self.0 { + match b { + // Common escape codes. + b'\t' => f.write_str("\\t")?, + b'\n' => f.write_str("\\n")?, + b'\r' => f.write_str("\\r")?, + // Printable characters. + 0x20..=0x7e => f.write_char(b as char)?, + _ => write!(f, "\\x{:02x}", b)?, + } + } + Ok(()) + } +} + +impl fmt::Debug for BStr { + /// Formats printable ASCII characters with a double quote on either end, + /// escaping the rest. + /// + /// ``` + /// # use kernel::{fmt, b_str, str::{BStr, CString}}; + /// // Embedded double quotes are escaped. + /// let ascii = b_str!("Hello, \"BStr\"!"); + /// let s = CString::try_from_fmt(fmt!("{:?}", ascii)).unwrap(); + /// assert_eq!(s.as_bytes(), "\"Hello, \\\"BStr\\\"!\"".as_bytes()); + /// + /// let non_ascii = b_str!("😺"); + /// let s = CString::try_from_fmt(fmt!("{:?}", non_ascii)).unwrap(); + /// assert_eq!(s.as_bytes(), "\"\\xf0\\x9f\\x98\\xba\"".as_bytes()); + /// ``` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_char('"')?; + for &b in &self.0 { + match b { + // Common escape codes. + b'\t' => f.write_str("\\t")?, + b'\n' => f.write_str("\\n")?, + b'\r' => f.write_str("\\r")?, + // String escape characters. + b'\"' => f.write_str("\\\"")?, + b'\\' => f.write_str("\\\\")?, + // Printable characters. + 0x20..=0x7e => f.write_char(b as char)?, + _ => write!(f, "\\x{:02x}", b)?, + } + } + f.write_char('"') + } +} + +impl Deref for BStr { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} /// Creates a new [`BStr`] from a string literal. /// @@ -33,7 +123,7 @@ pub type BStr = [u8]; macro_rules! b_str { ($str:literal) => {{ const S: &'static str = $str; - const C: &'static $crate::str::BStr = S.as_bytes(); + const C: &'static $crate::str::BStr = $crate::str::BStr::from_bytes(S.as_bytes()); C }}; } @@ -143,19 +233,32 @@ impl CStr { unsafe { core::mem::transmute(bytes) } } + /// Creates a mutable [`CStr`] from a `[u8]` without performing any + /// additional checks. + /// + /// # Safety + /// + /// `bytes` *must* end with a `NUL` byte, and should only have a single + /// `NUL` byte (or the string will be truncated). + #[inline] + pub unsafe fn from_bytes_with_nul_unchecked_mut(bytes: &mut [u8]) -> &mut CStr { + // SAFETY: Properties of `bytes` guaranteed by the safety precondition. + unsafe { &mut *(bytes as *mut [u8] as *mut CStr) } + } + /// Returns a C pointer to the string. #[inline] pub const fn as_char_ptr(&self) -> *const core::ffi::c_char { self.0.as_ptr() as _ } - /// Convert the string to a byte slice without the trailing 0 byte. + /// Convert the string to a byte slice without the trailing `NUL` byte. #[inline] pub fn as_bytes(&self) -> &[u8] { &self.0[..self.len()] } - /// Convert the string to a byte slice containing the trailing 0 byte. + /// Convert the string to a byte slice containing the trailing `NUL` byte. #[inline] pub const fn as_bytes_with_nul(&self) -> &[u8] { &self.0 @@ -191,9 +294,9 @@ impl CStr { /// ``` /// # use kernel::c_str; /// # use kernel::str::CStr; + /// let bar = c_str!("ツ"); /// // SAFETY: String literals are guaranteed to be valid UTF-8 /// // by the Rust compiler. - /// let bar = c_str!("ツ"); /// assert_eq!(unsafe { bar.as_str_unchecked() }, "ツ"); /// ``` #[inline] @@ -206,6 +309,70 @@ impl CStr { pub fn to_cstring(&self) -> Result<CString, AllocError> { CString::try_from(self) } + + /// Converts this [`CStr`] to its ASCII lower case equivalent in-place. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new lowercased value without modifying the existing one, use + /// [`to_ascii_lowercase()`]. + /// + /// [`to_ascii_lowercase()`]: #method.to_ascii_lowercase + pub fn make_ascii_lowercase(&mut self) { + // INVARIANT: This doesn't introduce or remove NUL bytes in the C + // string. + self.0.make_ascii_lowercase(); + } + + /// Converts this [`CStr`] to its ASCII upper case equivalent in-place. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To return a new uppercased value without modifying the existing one, use + /// [`to_ascii_uppercase()`]. + /// + /// [`to_ascii_uppercase()`]: #method.to_ascii_uppercase + pub fn make_ascii_uppercase(&mut self) { + // INVARIANT: This doesn't introduce or remove NUL bytes in the C + // string. + self.0.make_ascii_uppercase(); + } + + /// Returns a copy of this [`CString`] where each character is mapped to its + /// ASCII lower case equivalent. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To lowercase the value in-place, use [`make_ascii_lowercase`]. + /// + /// [`make_ascii_lowercase`]: str::make_ascii_lowercase + pub fn to_ascii_lowercase(&self) -> Result<CString, AllocError> { + let mut s = self.to_cstring()?; + + s.make_ascii_lowercase(); + + Ok(s) + } + + /// Returns a copy of this [`CString`] where each character is mapped to its + /// ASCII upper case equivalent. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To uppercase the value in-place, use [`make_ascii_uppercase`]. + /// + /// [`make_ascii_uppercase`]: str::make_ascii_uppercase + pub fn to_ascii_uppercase(&self) -> Result<CString, AllocError> { + let mut s = self.to_cstring()?; + + s.make_ascii_uppercase(); + + Ok(s) + } } impl fmt::Display for CStr { @@ -271,7 +438,7 @@ impl fmt::Debug for CStr { impl AsRef<BStr> for CStr { #[inline] fn as_ref(&self) -> &BStr { - self.as_bytes() + BStr::from_bytes(self.as_bytes()) } } @@ -280,7 +447,7 @@ impl Deref for CStr { #[inline] fn deref(&self) -> &Self::Target { - self.as_bytes() + self.as_ref() } } @@ -327,7 +494,7 @@ where #[inline] fn index(&self, index: Idx) -> &Self::Output { - &self.as_bytes()[index] + &self.as_ref()[index] } } @@ -357,6 +524,21 @@ macro_rules! c_str { #[cfg(test)] mod tests { use super::*; + use alloc::format; + + const ALL_ASCII_CHARS: &'static str = + "\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\x09\\x0a\\x0b\\x0c\\x0d\\x0e\\x0f\ + \\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a\\x1b\\x1c\\x1d\\x1e\\x1f \ + !\"#$%&'()*+,-./0123456789:;<=>?@\ + ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f\ + \\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d\\x8e\\x8f\ + \\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b\\x9c\\x9d\\x9e\\x9f\ + \\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9\\xaa\\xab\\xac\\xad\\xae\\xaf\ + \\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\ + \\xc0\\xc1\\xc2\\xc3\\xc4\\xc5\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\ + \\xd0\\xd1\\xd2\\xd3\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\ + \\xe0\\xe1\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef\ + \\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd\\xfe\\xff"; #[test] fn test_cstr_to_str() { @@ -381,6 +563,69 @@ mod tests { let unchecked_str = unsafe { checked_cstr.as_str_unchecked() }; assert_eq!(unchecked_str, "🐧"); } + + #[test] + fn test_cstr_display() { + let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap(); + assert_eq!(format!("{}", hello_world), "hello, world!"); + let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap(); + assert_eq!(format!("{}", non_printables), "\\x01\\x09\\x0a"); + let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap(); + assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu"); + let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap(); + assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80"); + } + + #[test] + fn test_cstr_display_all_bytes() { + let mut bytes: [u8; 256] = [0; 256]; + // fill `bytes` with [1..=255] + [0] + for i in u8::MIN..=u8::MAX { + bytes[i as usize] = i.wrapping_add(1); + } + let cstr = CStr::from_bytes_with_nul(&bytes).unwrap(); + assert_eq!(format!("{}", cstr), ALL_ASCII_CHARS); + } + + #[test] + fn test_cstr_debug() { + let hello_world = CStr::from_bytes_with_nul(b"hello, world!\0").unwrap(); + assert_eq!(format!("{:?}", hello_world), "\"hello, world!\""); + let non_printables = CStr::from_bytes_with_nul(b"\x01\x09\x0a\0").unwrap(); + assert_eq!(format!("{:?}", non_printables), "\"\\x01\\x09\\x0a\""); + let non_ascii = CStr::from_bytes_with_nul(b"d\xe9j\xe0 vu\0").unwrap(); + assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\""); + let good_bytes = CStr::from_bytes_with_nul(b"\xf0\x9f\xa6\x80\0").unwrap(); + assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\""); + } + + #[test] + fn test_bstr_display() { + let hello_world = BStr::from_bytes(b"hello, world!"); + assert_eq!(format!("{}", hello_world), "hello, world!"); + let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); + assert_eq!(format!("{}", escapes), "_\\t_\\n_\\r_\\_'_\"_"); + let others = BStr::from_bytes(b"\x01"); + assert_eq!(format!("{}", others), "\\x01"); + let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); + assert_eq!(format!("{}", non_ascii), "d\\xe9j\\xe0 vu"); + let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); + assert_eq!(format!("{}", good_bytes), "\\xf0\\x9f\\xa6\\x80"); + } + + #[test] + fn test_bstr_debug() { + let hello_world = BStr::from_bytes(b"hello, world!"); + assert_eq!(format!("{:?}", hello_world), "\"hello, world!\""); + let escapes = BStr::from_bytes(b"_\t_\n_\r_\\_\'_\"_"); + assert_eq!(format!("{:?}", escapes), "\"_\\t_\\n_\\r_\\\\_'_\\\"_\""); + let others = BStr::from_bytes(b"\x01"); + assert_eq!(format!("{:?}", others), "\"\\x01\""); + let non_ascii = BStr::from_bytes(b"d\xe9j\xe0 vu"); + assert_eq!(format!("{:?}", non_ascii), "\"d\\xe9j\\xe0 vu\""); + let good_bytes = BStr::from_bytes(b"\xf0\x9f\xa6\x80"); + assert_eq!(format!("{:?}", good_bytes), "\"\\xf0\\x9f\\xa6\\x80\""); + } } /// Allows formatting of [`fmt::Arguments`] into a raw buffer. @@ -449,7 +694,7 @@ impl RawFormatter { self.pos as _ } - /// Return the number of bytes written to the formatter. + /// Returns the number of bytes written to the formatter. pub(crate) fn bytes_written(&self) -> usize { self.pos - self.beg } @@ -558,7 +803,7 @@ impl CString { let size = f.bytes_written(); // Allocate a vector with the required number of bytes, and write to it. - let mut buf = Vec::try_with_capacity(size)?; + let mut buf = <Vec<_> as VecExt<_>>::with_capacity(size, GFP_KERNEL)?; // SAFETY: The buffer stored in `buf` is at least of size `size` and is valid for writes. let mut f = unsafe { Formatter::from_buffer(buf.as_mut_ptr(), size) }; f.write_fmt(args)?; @@ -593,13 +838,21 @@ impl Deref for CString { } } +impl DerefMut for CString { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: A `CString` is always NUL-terminated and contains no other + // NUL bytes. + unsafe { CStr::from_bytes_with_nul_unchecked_mut(self.buf.as_mut_slice()) } + } +} + impl<'a> TryFrom<&'a CStr> for CString { type Error = AllocError; fn try_from(cstr: &'a CStr) -> Result<CString, AllocError> { let mut buf = Vec::new(); - buf.try_extend_from_slice(cstr.as_bytes_with_nul()) + <Vec<_> as VecExt<_>>::extend_from_slice(&mut buf, cstr.as_bytes_with_nul(), GFP_KERNEL) .map_err(|_| AllocError)?; // INVARIANT: The `CStr` and `CString` types have the same invariants for @@ -608,6 +861,12 @@ impl<'a> TryFrom<&'a CStr> for CString { } } +impl fmt::Debug for CString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + /// A convenience alias for [`core::format_args`]. #[macro_export] macro_rules! fmt { diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index d219ee518eff..0ab20975a3b5 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -13,8 +13,9 @@ pub mod lock; mod locked_by; pub use arc::{Arc, ArcBorrow, UniqueArc}; -pub use condvar::CondVar; -pub use lock::{mutex::Mutex, spinlock::SpinLock}; +pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult}; +pub use lock::mutex::{new_mutex, Mutex}; +pub use lock::spinlock::{new_spinlock, SpinLock}; pub use locked_by::LockedBy; /// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. @@ -36,6 +37,12 @@ impl LockClassKey { } } +impl Default for LockClassKey { + fn default() -> Self { + Self::new() + } +} + /// Defines a new static lock class and returns a pointer to it. #[doc(hidden)] #[macro_export] diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs index 77cdbcf7bd2e..3673496c2363 100644 --- a/rust/kernel/sync/arc.rs +++ b/rust/kernel/sync/arc.rs @@ -16,7 +16,7 @@ //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html use crate::{ - bindings, + alloc::{box_ext::BoxExt, AllocError, Flags}, error::{self, Error}, init::{self, InPlaceInit, Init, PinInit}, try_init, @@ -24,13 +24,13 @@ use crate::{ }; use alloc::boxed::Box; use core::{ - alloc::{AllocError, Layout}, + alloc::Layout, fmt, marker::{PhantomData, Unsize}, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, pin::Pin, - ptr::{NonNull, Pointee}, + ptr::NonNull, }; use macros::pin_data; @@ -56,8 +56,8 @@ mod std_vendor; /// b: u32, /// } /// -/// // Create a ref-counted instance of `Example`. -/// let obj = Arc::try_new(Example { a: 10, b: 20 })?; +/// // Create a refcounted instance of `Example`. +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// /// // Get a new pointer to `obj` and increment the refcount. /// let cloned = obj.clone(); @@ -96,7 +96,7 @@ mod std_vendor; /// } /// } /// -/// let obj = Arc::try_new(Example { a: 10, b: 20 })?; +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// obj.use_reference(); /// obj.take_over(); /// # Ok::<(), Error>(()) @@ -119,7 +119,7 @@ mod std_vendor; /// impl MyTrait for Example {} /// /// // `obj` has type `Arc<Example>`. -/// let obj: Arc<Example> = Arc::try_new(Example)?; +/// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?; /// /// // `coerced` has type `Arc<dyn MyTrait>`. /// let coerced: Arc<dyn MyTrait> = obj; @@ -137,6 +137,39 @@ struct ArcInner<T: ?Sized> { data: T, } +impl<T: ?Sized> ArcInner<T> { + /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must + /// not yet have been destroyed. + unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { + let refcount_layout = Layout::new::<bindings::refcount_t>(); + // SAFETY: The caller guarantees that the pointer is valid. + let val_layout = Layout::for_value(unsafe { &*ptr }); + // SAFETY: We're computing the layout of a real struct that existed when compiling this + // binary, so its layout is not so large that it can trigger arithmetic overflow. + let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; + + // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and + // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field. + // + // This is documented at: + // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>. + let ptr = ptr as *const ArcInner<T>; + + // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the + // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is + // still valid. + let ptr = unsafe { ptr.byte_sub(val_offset) }; + + // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null + // address. + unsafe { NonNull::new_unchecked(ptr.cast_mut()) } + } +} + // This is to allow [`Arc`] (and variants) to be used as the type of `self`. impl<T: ?Sized> core::ops::Receiver for Arc<T> {} @@ -162,7 +195,7 @@ unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {} impl<T> Arc<T> { /// Constructs a new reference counted instance of `T`. - pub fn try_new(contents: T) -> Result<Self, AllocError> { + pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. let value = ArcInner { // SAFETY: There are no safety requirements for this FFI call. @@ -170,7 +203,7 @@ impl<T> Arc<T> { data: contents, }; - let inner = Box::try_new(value)?; + let inner = <Box<_> as BoxExt<_>>::new(value, flags)?; // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new // `Arc` object. @@ -181,22 +214,22 @@ impl<T> Arc<T> { /// /// If `T: !Unpin` it will not be able to move afterwards. #[inline] - pub fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Self> + pub fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self> where Error: From<E>, { - UniqueArc::pin_init(init).map(|u| u.into()) + UniqueArc::pin_init(init, flags).map(|u| u.into()) } /// Use the given initializer to in-place initialize a `T`. /// /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned. #[inline] - pub fn init<E>(init: impl Init<T, E>) -> error::Result<Self> + pub fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> where Error: From<E>, { - UniqueArc::init(init).map(|u| u.into()) + UniqueArc::init(init, flags).map(|u| u.into()) } } @@ -232,29 +265,13 @@ impl<T: ?Sized> Arc<T> { /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it /// must not be called more than once for each previous call to [`Arc::into_raw`]. pub unsafe fn from_raw(ptr: *const T) -> Self { - let refcount_layout = Layout::new::<bindings::refcount_t>(); - // SAFETY: The caller guarantees that the pointer is valid. - let val_layout = Layout::for_value(unsafe { &*ptr }); - // SAFETY: We're computing the layout of a real struct that existed when compiling this - // binary, so its layout is not so large that it can trigger arithmetic overflow. - let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; - - let metadata: <T as Pointee>::Metadata = core::ptr::metadata(ptr); - // SAFETY: The metadata of `T` and `ArcInner<T>` is the same because `ArcInner` is a struct - // with `T` as its last field. - // - // This is documented at: - // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>. - let metadata: <ArcInner<T> as Pointee>::Metadata = - unsafe { core::mem::transmute_copy(&metadata) }; - // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the - // pointer, since it originates from a previous call to `Arc::into_raw` and is still valid. - let ptr = unsafe { (ptr as *mut u8).sub(val_offset) as *mut () }; - let ptr = core::ptr::from_raw_parts_mut(ptr, metadata); + // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an + // `Arc` that is still valid. + let ptr = unsafe { ArcInner::container_of(ptr) }; // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the // reference count held then will be owned by the new `Arc` object. - unsafe { Self::from_inner(NonNull::new_unchecked(ptr)) } + unsafe { Self::from_inner(ptr) } } /// Returns an [`ArcBorrow`] from the given [`Arc`]. @@ -273,6 +290,68 @@ impl<T: ?Sized> Arc<T> { pub fn ptr_eq(this: &Self, other: &Self) -> bool { core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr()) } + + /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. + /// + /// When this destroys the `Arc`, it does so while properly avoiding races. This means that + /// this method will never call the destructor of the value. + /// + /// # Examples + /// + /// ``` + /// use kernel::sync::{Arc, UniqueArc}; + /// + /// let arc = Arc::new(42, GFP_KERNEL)?; + /// let unique_arc = arc.into_unique_or_drop(); + /// + /// // The above conversion should succeed since refcount of `arc` is 1. + /// assert!(unique_arc.is_some()); + /// + /// assert_eq!(*(unique_arc.unwrap()), 42); + /// + /// # Ok::<(), Error>(()) + /// ``` + /// + /// ``` + /// use kernel::sync::{Arc, UniqueArc}; + /// + /// let arc = Arc::new(42, GFP_KERNEL)?; + /// let another = arc.clone(); + /// + /// let unique_arc = arc.into_unique_or_drop(); + /// + /// // The above conversion should fail since refcount of `arc` is >1. + /// assert!(unique_arc.is_none()); + /// + /// # Ok::<(), Error>(()) + /// ``` + pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> { + // We will manually manage the refcount in this method, so we disable the destructor. + let me = ManuallyDrop::new(self); + // SAFETY: We own a refcount, so the pointer is still valid. + let refcount = unsafe { me.ptr.as_ref() }.refcount.get(); + + // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will + // return without further touching the `Arc`. If the refcount reaches zero, then there are + // no other arcs, and we can create a `UniqueArc`. + // + // SAFETY: We own a refcount, so the pointer is not dangling. + let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; + if is_zero { + // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized + // accesses to the refcount. + unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) }; + + // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We + // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin + // their values. + Some(Pin::from(UniqueArc { + inner: ManuallyDrop::into_inner(me), + })) + } else { + None + } + } } impl<T: 'static> ForeignOwnable for Arc<T> { @@ -365,12 +444,12 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// A borrowed reference to an [`Arc`] instance. /// /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler -/// to use just `&T`, which we can trivially get from an `Arc<T>` instance. +/// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance. /// /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>` /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference) -/// to a pointer (`Arc<T>`) to the object (`T`). An [`ArcBorrow`] eliminates this double -/// indirection while still allowing one to increment the refcount and getting an `Arc<T>` when/if +/// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double +/// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if /// needed. /// /// # Invariants @@ -389,7 +468,7 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// e.into() /// } /// -/// let obj = Arc::try_new(Example)?; +/// let obj = Arc::new(Example, GFP_KERNEL)?; /// let cloned = do_something(obj.as_arc_borrow()); /// /// // Assert that both `obj` and `cloned` point to the same underlying object. @@ -413,7 +492,7 @@ impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { /// } /// } /// -/// let obj = Arc::try_new(Example { a: 10, b: 20 })?; +/// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// obj.as_arc_borrow().use_reference(); /// # Ok::<(), Error>(()) /// ``` @@ -455,6 +534,27 @@ impl<T: ?Sized> ArcBorrow<'_, T> { _p: PhantomData, } } + + /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with + /// [`Arc::into_raw`]. + /// + /// # Safety + /// + /// * The provided pointer must originate from a call to [`Arc::into_raw`]. + /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must + /// not hit zero. + /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a + /// [`UniqueArc`] reference to this value. + pub unsafe fn from_raw(ptr: *const T) -> Self { + // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an + // `Arc` that is still valid. + let ptr = unsafe { ArcInner::container_of(ptr) }; + + // SAFETY: The caller promises that the value remains valid since the reference count must + // not hit zero, and no mutable reference will be created since that would involve a + // `UniqueArc`. + unsafe { Self::new(ptr) } + } } impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> { @@ -501,7 +601,7 @@ impl<T: ?Sized> Deref for ArcBorrow<'_, T> { /// } /// /// fn test() -> Result<Arc<Example>> { -/// let mut x = UniqueArc::try_new(Example { a: 10, b: 20 })?; +/// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; /// x.a += 1; /// x.b += 1; /// Ok(x.into()) @@ -510,7 +610,7 @@ impl<T: ?Sized> Deref for ArcBorrow<'_, T> { /// # test().unwrap(); /// ``` /// -/// In the following example we first allocate memory for a ref-counted `Example` but we don't +/// In the following example we first allocate memory for a refcounted `Example` but we don't /// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`], /// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens /// in one context (e.g., sleepable) and initialisation in another (e.g., atomic): @@ -524,7 +624,7 @@ impl<T: ?Sized> Deref for ArcBorrow<'_, T> { /// } /// /// fn test() -> Result<Arc<Example>> { -/// let x = UniqueArc::try_new_uninit()?; +/// let x = UniqueArc::new_uninit(GFP_KERNEL)?; /// Ok(x.write(Example { a: 10, b: 20 }).into()) /// } /// @@ -544,7 +644,7 @@ impl<T: ?Sized> Deref for ArcBorrow<'_, T> { /// } /// /// fn test() -> Result<Arc<Example>> { -/// let mut pinned = Pin::from(UniqueArc::try_new(Example { a: 10, b: 20 })?); +/// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?); /// // We can modify `pinned` because it is `Unpin`. /// pinned.as_mut().a += 1; /// Ok(pinned.into()) @@ -558,23 +658,26 @@ pub struct UniqueArc<T: ?Sized> { impl<T> UniqueArc<T> { /// Tries to allocate a new [`UniqueArc`] instance. - pub fn try_new(value: T) -> Result<Self, AllocError> { + pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> { Ok(Self { - // INVARIANT: The newly-created object has a ref-count of 1. - inner: Arc::try_new(value)?, + // INVARIANT: The newly-created object has a refcount of 1. + inner: Arc::new(value, flags)?, }) } /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. - pub fn try_new_uninit() -> Result<UniqueArc<MaybeUninit<T>>, AllocError> { + pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> { // INVARIANT: The refcount is initialised to a non-zero value. - let inner = Box::try_init::<AllocError>(try_init!(ArcInner { - // SAFETY: There are no safety requirements for this FFI call. - refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), - data <- init::uninit::<T, AllocError>(), - }? AllocError))?; + let inner = Box::try_init::<AllocError>( + try_init!(ArcInner { + // SAFETY: There are no safety requirements for this FFI call. + refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), + data <- init::uninit::<T, AllocError>(), + }? AllocError), + flags, + )?; Ok(UniqueArc { - // INVARIANT: The newly-created object has a ref-count of 1. + // INVARIANT: The newly-created object has a refcount of 1. // SAFETY: The pointer from the `Box` is valid. inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, }) diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs index b679b6f6dbeb..2b306afbe56d 100644 --- a/rust/kernel/sync/condvar.rs +++ b/rust/kernel/sync/condvar.rs @@ -6,8 +6,17 @@ //! variable. use super::{lock::Backend, lock::Guard, LockClassKey}; -use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque}; +use crate::{ + init::PinInit, + pin_init, + str::CStr, + task::{MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE, TASK_NORMAL, TASK_UNINTERRUPTIBLE}, + time::Jiffies, + types::Opaque, +}; +use core::ffi::{c_int, c_long}; use core::marker::PhantomPinned; +use core::ptr; use macros::pin_data; /// Creates a [`CondVar`] initialiser with the given name and a newly-created lock class. @@ -17,6 +26,7 @@ macro_rules! new_condvar { $crate::sync::CondVar::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } +pub use new_condvar; /// A conditional variable. /// @@ -34,8 +44,7 @@ macro_rules! new_condvar { /// The following is an example of using a condvar with a mutex: /// /// ``` -/// use kernel::sync::{CondVar, Mutex}; -/// use kernel::{new_condvar, new_mutex}; +/// use kernel::sync::{new_condvar, new_mutex, CondVar, Mutex}; /// /// #[pin_data] /// pub struct Example { @@ -50,7 +59,7 @@ macro_rules! new_condvar { /// fn wait_for_value(e: &Example, v: u32) { /// let mut guard = e.value.lock(); /// while *guard != v { -/// e.value_changed.wait_uninterruptible(&mut guard); +/// e.value_changed.wait(&mut guard); /// } /// } /// @@ -65,18 +74,20 @@ macro_rules! new_condvar { /// Box::pin_init(pin_init!(Example { /// value <- new_mutex!(0), /// value_changed <- new_condvar!(), -/// })) +/// }), GFP_KERNEL) /// } /// ``` /// -/// [`struct wait_queue_head`]: ../../../include/linux/wait.h +/// [`struct wait_queue_head`]: srctree/include/linux/wait.h #[pin_data] pub struct CondVar { #[pin] - pub(crate) wait_list: Opaque<bindings::wait_queue_head>, + pub(crate) wait_queue_head: Opaque<bindings::wait_queue_head>, /// A condvar needs to be pinned because it contains a [`struct list_head`] that is /// self-referential, so it cannot be safely moved once it is initialised. + /// + /// [`struct list_head`]: srctree/include/linux/types.h #[pin] _pin: PhantomPinned, } @@ -96,71 +107,109 @@ impl CondVar { _pin: PhantomPinned, // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have // static lifetimes so they live indefinitely. - wait_list <- Opaque::ffi_init(|slot| unsafe { + wait_queue_head <- Opaque::ffi_init(|slot| unsafe { bindings::__init_waitqueue_head(slot, name.as_char_ptr(), key.as_ptr()) }), }) } - fn wait_internal<T: ?Sized, B: Backend>(&self, wait_state: u32, guard: &mut Guard<'_, T, B>) { + fn wait_internal<T: ?Sized, B: Backend>( + &self, + wait_state: c_int, + guard: &mut Guard<'_, T, B>, + timeout_in_jiffies: c_long, + ) -> c_long { let wait = Opaque::<bindings::wait_queue_entry>::uninit(); // SAFETY: `wait` points to valid memory. unsafe { bindings::init_wait(wait.get()) }; - // SAFETY: Both `wait` and `wait_list` point to valid memory. + // SAFETY: Both `wait` and `wait_queue_head` point to valid memory. unsafe { - bindings::prepare_to_wait_exclusive(self.wait_list.get(), wait.get(), wait_state as _) + bindings::prepare_to_wait_exclusive(self.wait_queue_head.get(), wait.get(), wait_state) }; - // SAFETY: No arguments, switches to another thread. - guard.do_unlocked(|| unsafe { bindings::schedule() }); + // SAFETY: Switches to another thread. The timeout can be any number. + let ret = guard.do_unlocked(|| unsafe { bindings::schedule_timeout(timeout_in_jiffies) }); + + // SAFETY: Both `wait` and `wait_queue_head` point to valid memory. + unsafe { bindings::finish_wait(self.wait_queue_head.get(), wait.get()) }; - // SAFETY: Both `wait` and `wait_list` point to valid memory. - unsafe { bindings::finish_wait(self.wait_list.get(), wait.get()) }; + ret } - /// Releases the lock and waits for a notification in interruptible mode. + /// Releases the lock and waits for a notification in uninterruptible mode. /// /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the /// thread to sleep, reacquiring the lock on wake up. It wakes up when notified by - /// [`CondVar::notify_one`] or [`CondVar::notify_all`], or when the thread receives a signal. - /// It may also wake up spuriously. + /// [`CondVar::notify_one`] or [`CondVar::notify_all`]. Note that it may also wake up + /// spuriously. + pub fn wait<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) { + self.wait_internal(TASK_UNINTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT); + } + + /// Releases the lock and waits for a notification in interruptible mode. + /// + /// Similar to [`CondVar::wait`], except that the wait is interruptible. That is, the thread may + /// wake up due to signals. It may also wake up spuriously. /// /// Returns whether there is a signal pending. - #[must_use = "wait returns if a signal is pending, so the caller must check the return value"] - pub fn wait<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) -> bool { - self.wait_internal(bindings::TASK_INTERRUPTIBLE, guard); + #[must_use = "wait_interruptible returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) -> bool { + self.wait_internal(TASK_INTERRUPTIBLE, guard, MAX_SCHEDULE_TIMEOUT); crate::current!().signal_pending() } - /// Releases the lock and waits for a notification in uninterruptible mode. + /// Releases the lock and waits for a notification in interruptible mode. /// - /// Similar to [`CondVar::wait`], except that the wait is not interruptible. That is, the - /// thread won't wake up due to signals. It may, however, wake up supirously. - pub fn wait_uninterruptible<T: ?Sized, B: Backend>(&self, guard: &mut Guard<'_, T, B>) { - self.wait_internal(bindings::TASK_UNINTERRUPTIBLE, guard) + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or + /// [`CondVar::notify_all`], or when a timeout occurs, or when the thread receives a signal. + #[must_use = "wait_interruptible_timeout returns if a signal is pending, so the caller must check the return value"] + pub fn wait_interruptible_timeout<T: ?Sized, B: Backend>( + &self, + guard: &mut Guard<'_, T, B>, + jiffies: Jiffies, + ) -> CondVarTimeoutResult { + let jiffies = jiffies.try_into().unwrap_or(MAX_SCHEDULE_TIMEOUT); + let res = self.wait_internal(TASK_INTERRUPTIBLE, guard, jiffies); + + match (res as Jiffies, crate::current!().signal_pending()) { + (jiffies, true) => CondVarTimeoutResult::Signal { jiffies }, + (0, false) => CondVarTimeoutResult::Timeout, + (jiffies, false) => CondVarTimeoutResult::Woken { jiffies }, + } } - /// Calls the kernel function to notify the appropriate number of threads with the given flags. - fn notify(&self, count: i32, flags: u32) { - // SAFETY: `wait_list` points to valid memory. + /// Calls the kernel function to notify the appropriate number of threads. + fn notify(&self, count: c_int) { + // SAFETY: `wait_queue_head` points to valid memory. unsafe { bindings::__wake_up( - self.wait_list.get(), - bindings::TASK_NORMAL, + self.wait_queue_head.get(), + TASK_NORMAL, count, - flags as _, + ptr::null_mut(), ) }; } + /// Calls the kernel function to notify one thread synchronously. + /// + /// This method behaves like `notify_one`, except that it hints to the scheduler that the + /// current thread is about to go to sleep, so it should schedule the target thread on the same + /// CPU. + pub fn notify_sync(&self) { + // SAFETY: `wait_queue_head` points to valid memory. + unsafe { bindings::__wake_up_sync(self.wait_queue_head.get(), TASK_NORMAL) }; + } + /// Wakes a single waiter up, if any. /// /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost /// completely (as opposed to automatically waking up the next waiter). pub fn notify_one(&self) { - self.notify(1, 0); + self.notify(1); } /// Wakes all waiters up, if any. @@ -168,6 +217,22 @@ impl CondVar { /// This is not 'sticky' in the sense that if no thread is waiting, the notification is lost /// completely (as opposed to automatically waking up the next waiter). pub fn notify_all(&self) { - self.notify(0, 0); + self.notify(0); } } + +/// The return type of `wait_timeout`. +pub enum CondVarTimeoutResult { + /// The timeout was reached. + Timeout, + /// Somebody woke us up. + Woken { + /// Remaining sleep duration. + jiffies: Jiffies, + }, + /// A signal occurred. + Signal { + /// Remaining sleep duration. + jiffies: Jiffies, + }, +} diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index f12a684bc957..f6c34ca4d819 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -6,7 +6,7 @@ //! spinlocks, raw spinlocks) to be provided with minimal effort. use super::LockClassKey; -use crate::{bindings, init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard}; +use crate::{init::PinInit, pin_init, str::CStr, types::Opaque, types::ScopeGuard}; use core::{cell::UnsafeCell, marker::PhantomData, marker::PhantomPinned}; use macros::pin_data; @@ -21,14 +21,21 @@ pub mod spinlock; /// # Safety /// /// - Implementers must ensure that only one thread/CPU may access the protected data once the lock -/// is owned, that is, between calls to `lock` and `unlock`. -/// - Implementers must also ensure that `relock` uses the same locking method as the original -/// lock operation. +/// is owned, that is, between calls to [`lock`] and [`unlock`]. +/// - Implementers must also ensure that [`relock`] uses the same locking method as the original +/// lock operation. +/// +/// [`lock`]: Backend::lock +/// [`unlock`]: Backend::unlock +/// [`relock`]: Backend::relock pub unsafe trait Backend { /// The state required by the lock. type State; - /// The state required to be kept between lock and unlock. + /// The state required to be kept between [`lock`] and [`unlock`]. + /// + /// [`lock`]: Backend::lock + /// [`unlock`]: Backend::unlock type GuardState; /// Initialises the lock. @@ -139,7 +146,7 @@ pub struct Guard<'a, T: ?Sized, B: Backend> { unsafe impl<T: Sync + ?Sized, B: Backend> Sync for Guard<'_, T, B> {} impl<T: ?Sized, B: Backend> Guard<'_, T, B> { - pub(crate) fn do_unlocked(&mut self, cb: impl FnOnce()) { + pub(crate) fn do_unlocked<U>(&mut self, cb: impl FnOnce() -> U) -> U { // SAFETY: The caller owns the lock, so it is safe to unlock it. unsafe { B::unlock(self.lock.state.get(), &self.state) }; @@ -147,7 +154,7 @@ impl<T: ?Sized, B: Backend> Guard<'_, T, B> { let _relock = ScopeGuard::new(|| unsafe { B::relock(self.lock.state.get(), &mut self.state) }); - cb(); + cb() } } diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 09276fedc091..30632070ee67 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -4,8 +4,6 @@ //! //! This module allows Rust code to use the kernel's `struct mutex`. -use crate::bindings; - /// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class. /// /// It uses the name if one is given, otherwise it generates one based on the file name and line @@ -17,6 +15,7 @@ macro_rules! new_mutex { $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } +pub use new_mutex; /// A mutual exclusion primitive. /// @@ -35,7 +34,7 @@ macro_rules! new_mutex { /// contains an inner struct (`Inner`) that is protected by a mutex. /// /// ``` -/// use kernel::{init::InPlaceInit, init::PinInit, new_mutex, pin_init, sync::Mutex}; +/// use kernel::sync::{new_mutex, Mutex}; /// /// struct Inner { /// a: u32, @@ -59,7 +58,7 @@ macro_rules! new_mutex { /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new())?; +/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); @@ -84,7 +83,7 @@ macro_rules! new_mutex { /// } /// ``` /// -/// [`struct mutex`]: ../../../../include/linux/mutex.h +/// [`struct mutex`]: srctree/include/linux/mutex.h pub type Mutex<T> = super::Lock<T, MutexBackend>; /// A kernel `struct mutex` lock backend. diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index 91eb2c9e9123..ea5c5bc1ce12 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -4,8 +4,6 @@ //! //! This module allows Rust code to use the kernel's `spinlock_t`. -use crate::bindings; - /// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. /// /// It uses the name if one is given, otherwise it generates one based on the file name and line @@ -17,6 +15,7 @@ macro_rules! new_spinlock { $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } +pub use new_spinlock; /// A spinlock. /// @@ -33,7 +32,7 @@ macro_rules! new_spinlock { /// contains an inner struct (`Inner`) that is protected by a spinlock. /// /// ``` -/// use kernel::{init::InPlaceInit, init::PinInit, new_spinlock, pin_init, sync::SpinLock}; +/// use kernel::sync::{new_spinlock, SpinLock}; /// /// struct Inner { /// a: u32, @@ -57,7 +56,7 @@ macro_rules! new_spinlock { /// } /// /// // Allocate a boxed `Example`. -/// let e = Box::pin_init(Example::new())?; +/// let e = Box::pin_init(Example::new(), GFP_KERNEL)?; /// assert_eq!(e.c, 10); /// assert_eq!(e.d.lock().a, 20); /// assert_eq!(e.d.lock().b, 30); @@ -82,7 +81,7 @@ macro_rules! new_spinlock { /// } /// ``` /// -/// [`spinlock_t`]: ../../../../include/linux/spinlock.h +/// [`spinlock_t`]: srctree/include/linux/spinlock.h pub type SpinLock<T> = super::Lock<T, SpinLockBackend>; /// A kernel `spinlock_t` lock backend. @@ -112,7 +111,7 @@ unsafe impl super::Backend for SpinLockBackend { unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the - // caller is the owner of the mutex. + // caller is the owner of the spinlock. unsafe { bindings::spin_unlock(ptr) } } } diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs index b17ee5cd98f3..babc731bd5f6 100644 --- a/rust/kernel/sync/locked_by.rs +++ b/rust/kernel/sync/locked_by.rs @@ -9,14 +9,17 @@ use core::{cell::UnsafeCell, mem::size_of, ptr}; /// Allows access to some data to be serialised by a lock that does not wrap it. /// /// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g., -/// [`super::Mutex`] or [`super::SpinLock`]. [`LockedBy`] is meant for cases when this is not -/// possible. For example, if a container has a lock and some data in the contained elements needs +/// [`Mutex`] or [`SpinLock`]. [`LockedBy`] is meant for cases when this is not possible. +/// For example, if a container has a lock and some data in the contained elements needs /// to be protected by the same lock. /// /// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it /// when the caller shows evidence that the 'external' lock is locked. It panics if the evidence /// refers to the wrong instance of the lock. /// +/// [`Mutex`]: super::Mutex +/// [`SpinLock`]: super::SpinLock +/// /// # Examples /// /// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an diff --git a/rust/kernel/task.rs b/rust/kernel/task.rs index b2299bc7ac1f..55dff7e088bf 100644 --- a/rust/kernel/task.rs +++ b/rust/kernel/task.rs @@ -2,10 +2,26 @@ //! Tasks (threads and processes). //! -//! C header: [`include/linux/sched.h`](../../../../include/linux/sched.h). - -use crate::{bindings, types::Opaque}; -use core::{marker::PhantomData, ops::Deref, ptr}; +//! C header: [`include/linux/sched.h`](srctree/include/linux/sched.h). + +use crate::types::Opaque; +use core::{ + ffi::{c_int, c_long, c_uint}, + marker::PhantomData, + ops::Deref, + ptr, +}; + +/// A sentinel value used for infinite timeouts. +pub const MAX_SCHEDULE_TIMEOUT: c_long = c_long::MAX; + +/// Bitmask for tasks that are sleeping in an interruptible state. +pub const TASK_INTERRUPTIBLE: c_int = bindings::TASK_INTERRUPTIBLE as c_int; +/// Bitmask for tasks that are sleeping in an uninterruptible state. +pub const TASK_UNINTERRUPTIBLE: c_int = bindings::TASK_UNINTERRUPTIBLE as c_int; +/// Convenience constant for waking up tasks regardless of whether they are in interruptible or +/// uninterruptible sleep. +pub const TASK_NORMAL: c_uint = bindings::TASK_NORMAL as c_uint; /// Returns the currently running task. #[macro_export] @@ -23,7 +39,7 @@ macro_rules! current { /// /// All instances are valid tasks created by the C portion of the kernel. /// -/// Instances of this type are always ref-counted, that is, a call to `get_task_struct` ensures +/// Instances of this type are always refcounted, that is, a call to `get_task_struct` ensures /// that the allocation remains valid at least until the matching call to `put_task_struct`. /// /// # Examples @@ -116,7 +132,7 @@ impl Task { /// Returns the group leader of the given task. pub fn group_leader(&self) -> &Task { // SAFETY: By the type invariant, we know that `self.0` is a valid task. Valid tasks always - // have a valid group_leader. + // have a valid `group_leader`. let ptr = unsafe { *ptr::addr_of!((*self.0.get()).group_leader) }; // SAFETY: The lifetime of the returned task reference is tied to the lifetime of `self`, @@ -147,7 +163,7 @@ impl Task { } } -// SAFETY: The type invariants guarantee that `Task` is always ref-counted. +// SAFETY: The type invariants guarantee that `Task` is always refcounted. unsafe impl crate::types::AlwaysRefCounted for Task { fn inc_ref(&self) { // SAFETY: The existence of a shared reference means that the refcount is nonzero. diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs new file mode 100644 index 000000000000..e3bb5e89f88d --- /dev/null +++ b/rust/kernel/time.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Time related primitives. +//! +//! This module contains the kernel APIs related to time and timers that +//! have been ported or wrapped for usage by Rust code in the kernel. +//! +//! C header: [`include/linux/jiffies.h`](srctree/include/linux/jiffies.h). +//! C header: [`include/linux/ktime.h`](srctree/include/linux/ktime.h). + +/// The number of nanoseconds per millisecond. +pub const NSEC_PER_MSEC: i64 = bindings::NSEC_PER_MSEC as i64; + +/// The time unit of Linux kernel. One jiffy equals (1/HZ) second. +pub type Jiffies = core::ffi::c_ulong; + +/// The millisecond time unit. +pub type Msecs = core::ffi::c_uint; + +/// Converts milliseconds to jiffies. +#[inline] +pub fn msecs_to_jiffies(msecs: Msecs) -> Jiffies { + // SAFETY: The `__msecs_to_jiffies` function is always safe to call no + // matter what the argument is. + unsafe { bindings::__msecs_to_jiffies(msecs) } +} + +/// A Rust wrapper around a `ktime_t`. +#[repr(transparent)] +#[derive(Copy, Clone)] +pub struct Ktime { + inner: bindings::ktime_t, +} + +impl Ktime { + /// Create a `Ktime` from a raw `ktime_t`. + #[inline] + pub fn from_raw(inner: bindings::ktime_t) -> Self { + Self { inner } + } + + /// Get the current time using `CLOCK_MONOTONIC`. + #[inline] + pub fn ktime_get() -> Self { + // SAFETY: It is always safe to call `ktime_get` outside of NMI context. + Self::from_raw(unsafe { bindings::ktime_get() }) + } + + /// Divide the number of nanoseconds by a compile-time constant. + #[inline] + fn divns_constant<const DIV: i64>(self) -> i64 { + self.to_ns() / DIV + } + + /// Returns the number of nanoseconds. + #[inline] + pub fn to_ns(self) -> i64 { + self.inner + } + + /// Returns the number of milliseconds. + #[inline] + pub fn to_ms(self) -> i64 { + self.divns_constant::<NSEC_PER_MSEC>() + } +} + +/// Returns the number of milliseconds between two ktimes. +#[inline] +pub fn ktime_ms_delta(later: Ktime, earlier: Ktime) -> i64 { + (later - earlier).to_ms() +} + +impl core::ops::Sub for Ktime { + type Output = Ktime; + + #[inline] + fn sub(self, other: Ktime) -> Ktime { + Self { + inner: self.inner - other.inner, + } + } +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index fdb778e65d79..2e7c9008621f 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -46,6 +46,25 @@ pub trait ForeignOwnable: Sized { /// Additionally, all instances (if any) of values returned by [`ForeignOwnable::borrow`] for /// this object must have been dropped. unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self; + + /// Tries to convert a foreign-owned object back to a Rust-owned one. + /// + /// A convenience wrapper over [`ForeignOwnable::from_foreign`] that returns [`None`] if `ptr` + /// is null. + /// + /// # Safety + /// + /// `ptr` must either be null or satisfy the safety requirements for + /// [`ForeignOwnable::from_foreign`]. + unsafe fn try_from_foreign(ptr: *const core::ffi::c_void) -> Option<Self> { + if ptr.is_null() { + None + } else { + // SAFETY: Since `ptr` is not null here, then `ptr` satisfies the safety requirements + // of `from_foreign` given the safety requirements of this function. + unsafe { Some(Self::from_foreign(ptr)) } + } + } } impl<T: 'static> ForeignOwnable for Box<T> { @@ -90,6 +109,7 @@ impl ForeignOwnable for () { /// /// In the example below, we have multiple exit paths and we want to log regardless of which one is /// taken: +/// /// ``` /// # use kernel::types::ScopeGuard; /// fn example1(arg: bool) { @@ -108,6 +128,7 @@ impl ForeignOwnable for () { /// /// In the example below, we want to log the same message on all early exits but a different one on /// the main exit path: +/// /// ``` /// # use kernel::types::ScopeGuard; /// fn example2(arg: bool) { @@ -129,17 +150,18 @@ impl ForeignOwnable for () { /// /// In the example below, we need a mutable object (the vector) to be accessible within the log /// function, so we wrap it in the [`ScopeGuard`]: +/// /// ``` /// # use kernel::types::ScopeGuard; /// fn example3(arg: bool) -> Result { /// let mut vec = /// ScopeGuard::new_with_data(Vec::new(), |v| pr_info!("vec had {} elements\n", v.len())); /// -/// vec.try_push(10u8)?; +/// vec.push(10u8, GFP_KERNEL)?; /// if arg { /// return Ok(()); /// } -/// vec.try_push(20u8)?; +/// vec.push(20u8, GFP_KERNEL)?; /// Ok(()) /// } /// @@ -248,7 +270,7 @@ impl<T> Opaque<T> { } /// Returns a raw pointer to the opaque data. - pub fn get(&self) -> *mut T { + pub const fn get(&self) -> *mut T { UnsafeCell::get(&self.value).cast::<T>() } diff --git a/rust/kernel/workqueue.rs b/rust/kernel/workqueue.rs index b67fb1ba168e..1cec63a2aea8 100644 --- a/rust/kernel/workqueue.rs +++ b/rust/kernel/workqueue.rs @@ -12,19 +12,19 @@ //! //! # The raw API //! -//! The raw API consists of the `RawWorkItem` trait, where the work item needs to provide an +//! The raw API consists of the [`RawWorkItem`] trait, where the work item needs to provide an //! arbitrary function that knows how to enqueue the work item. It should usually not be used //! directly, but if you want to, you can use it without using the pieces from the safe API. //! //! # The safe API //! -//! The safe API is used via the `Work` struct and `WorkItem` traits. Furthermore, it also includes -//! a trait called `WorkItemPointer`, which is usually not used directly by the user. +//! The safe API is used via the [`Work`] struct and [`WorkItem`] traits. Furthermore, it also +//! includes a trait called [`WorkItemPointer`], which is usually not used directly by the user. //! -//! * The `Work` struct is the Rust wrapper for the C `work_struct` type. -//! * The `WorkItem` trait is implemented for structs that can be enqueued to a workqueue. -//! * The `WorkItemPointer` trait is implemented for the pointer type that points at a something -//! that implements `WorkItem`. +//! * The [`Work`] struct is the Rust wrapper for the C `work_struct` type. +//! * The [`WorkItem`] trait is implemented for structs that can be enqueued to a workqueue. +//! * The [`WorkItemPointer`] trait is implemented for the pointer type that points at a something +//! that implements [`WorkItem`]. //! //! ## Example //! @@ -33,10 +33,8 @@ //! we do not need to specify ids for the fields. //! //! ``` -//! use kernel::prelude::*; //! use kernel::sync::Arc; -//! use kernel::workqueue::{self, Work, WorkItem}; -//! use kernel::{impl_has_work, new_work}; +//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; //! //! #[pin_data] //! struct MyStruct { @@ -54,7 +52,7 @@ //! Arc::pin_init(pin_init!(MyStruct { //! value, //! work <- new_work!("MyStruct::work"), -//! })) +//! }), GFP_KERNEL) //! } //! } //! @@ -76,10 +74,8 @@ //! The following example shows how multiple `work_struct` fields can be used: //! //! ``` -//! use kernel::prelude::*; //! use kernel::sync::Arc; -//! use kernel::workqueue::{self, Work, WorkItem}; -//! use kernel::{impl_has_work, new_work}; +//! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; //! //! #[pin_data] //! struct MyStruct { @@ -103,7 +99,7 @@ //! value_2, //! work_1 <- new_work!("MyStruct::work_1"), //! work_2 <- new_work!("MyStruct::work_2"), -//! })) +//! }), GFP_KERNEL) //! } //! } //! @@ -132,13 +128,11 @@ //! } //! ``` //! -//! C header: [`include/linux/workqueue.h`](../../../../include/linux/workqueue.h) +//! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) -use crate::{bindings, prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; -use alloc::alloc::AllocError; -use alloc::boxed::Box; +use crate::alloc::{AllocError, Flags}; +use crate::{prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; use core::marker::PhantomData; -use core::pin::Pin; /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. #[macro_export] @@ -147,6 +141,7 @@ macro_rules! new_work { $crate::workqueue::Work::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) }; } +pub use new_work; /// A kernel work queue. /// @@ -168,7 +163,7 @@ impl Queue { /// # Safety /// /// The caller must ensure that the provided raw pointer is not dangling, that it points at a - /// valid workqueue, and that it remains valid until the end of 'a. + /// valid workqueue, and that it remains valid until the end of `'a`. pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue { // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The // caller promises that the pointer is not dangling. @@ -199,7 +194,11 @@ impl Queue { // stay valid until we call the function pointer in the `work_struct`, so the access is ok. unsafe { w.__enqueue(move |work_ptr| { - bindings::queue_work_on(bindings::WORK_CPU_UNBOUND as _, queue_ptr, work_ptr) + bindings::queue_work_on( + bindings::wq_misc_consts_WORK_CPU_UNBOUND as _, + queue_ptr, + work_ptr, + ) }) } } @@ -207,18 +206,24 @@ impl Queue { /// Tries to spawn the given function or closure as a work item. /// /// This method can fail because it allocates memory to store the work item. - pub fn try_spawn<T: 'static + Send + FnOnce()>(&self, func: T) -> Result<(), AllocError> { + pub fn try_spawn<T: 'static + Send + FnOnce()>( + &self, + flags: Flags, + func: T, + ) -> Result<(), AllocError> { let init = pin_init!(ClosureWork { work <- new_work!("Queue::try_spawn"), func: Some(func), }); - self.enqueue(Box::pin_init(init).map_err(|_| AllocError)?); + self.enqueue(Box::pin_init(init, flags).map_err(|_| AllocError)?); Ok(()) } } -/// A helper type used in `try_spawn`. +/// A helper type used in [`try_spawn`]. +/// +/// [`try_spawn`]: Queue::try_spawn #[pin_data] struct ClosureWork<T> { #[pin] @@ -253,14 +258,16 @@ impl<T: FnOnce()> WorkItem for ClosureWork<T> { /// actual value of the id is not important as long as you use different ids for different fields /// of the same struct. (Fields of different structs need not use different ids.) /// -/// Note that the id is used only to select the right method to call during compilation. It wont be +/// Note that the id is used only to select the right method to call during compilation. It won't be /// part of the final executable. /// /// # Safety /// -/// Implementers must ensure that any pointers passed to a `queue_work_on` closure by `__enqueue` +/// Implementers must ensure that any pointers passed to a `queue_work_on` closure by [`__enqueue`] /// remain valid for the duration specified in the guarantees section of the documentation for -/// `__enqueue`. +/// [`__enqueue`]. +/// +/// [`__enqueue`]: RawWorkItem::__enqueue pub unsafe trait RawWorkItem<const ID: u64> { /// The return type of [`Queue::enqueue`]. type EnqueueOutput; @@ -290,10 +297,11 @@ pub unsafe trait RawWorkItem<const ID: u64> { /// Defines the method that should be called directly when a work item is executed. /// -/// This trait is implemented by `Pin<Box<T>>` and `Arc<T>`, and is mainly intended to be +/// This trait is implemented by `Pin<Box<T>>` and [`Arc<T>`], and is mainly intended to be /// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] -/// instead. The `run` method on this trait will usually just perform the appropriate -/// `container_of` translation and then call into the `run` method from the [`WorkItem`] trait. +/// instead. The [`run`] method on this trait will usually just perform the appropriate +/// `container_of` translation and then call into the [`run`][WorkItem::run] method from the +/// [`WorkItem`] trait. /// /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. /// @@ -309,8 +317,10 @@ pub unsafe trait WorkItemPointer<const ID: u64>: RawWorkItem<ID> { /// /// # Safety /// - /// The provided `work_struct` pointer must originate from a previous call to `__enqueue` where - /// the `queue_work_on` closure returned true, and the pointer must still be valid. + /// The provided `work_struct` pointer must originate from a previous call to [`__enqueue`] + /// where the `queue_work_on` closure returned true, and the pointer must still be valid. + /// + /// [`__enqueue`]: RawWorkItem::__enqueue unsafe extern "C" fn run(ptr: *mut bindings::work_struct); } @@ -328,14 +338,18 @@ pub trait WorkItem<const ID: u64 = 0> { /// Links for a work item. /// -/// This struct contains a function pointer to the `run` function from the [`WorkItemPointer`] +/// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] /// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue. /// /// Wraps the kernel's C `struct work_struct`. /// /// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it. +/// +/// [`run`]: WorkItemPointer::run +#[pin_data] #[repr(transparent)] pub struct Work<T: ?Sized, const ID: u64 = 0> { + #[pin] work: Opaque<bindings::work_struct>, _inner: PhantomData<T>, } @@ -357,21 +371,22 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> { where T: WorkItem<ID>, { - // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as the work - // item function. - unsafe { - kernel::init::pin_init_from_closure(move |slot| { - let slot = Self::raw_get(slot); - bindings::init_work_with_key( - slot, - Some(T::Pointer::run), - false, - name.as_char_ptr(), - key.as_ptr(), - ); - Ok(()) - }) - } + pin_init!(Self { + work <- Opaque::ffi_init(|slot| { + // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as + // the work item function. + unsafe { + bindings::init_work_with_key( + slot, + Some(T::Pointer::run), + false, + name.as_char_ptr(), + key.as_ptr(), + ) + } + }), + _inner: PhantomData, + }) } /// Get a pointer to the inner `work_struct`. @@ -396,9 +411,7 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> { /// like this: /// /// ```no_run -/// use kernel::impl_has_work; -/// use kernel::prelude::*; -/// use kernel::workqueue::Work; +/// use kernel::workqueue::{impl_has_work, Work}; /// /// struct MyWorkItem { /// work_field: Work<MyWorkItem, 1>, @@ -409,28 +422,25 @@ impl<T: ?Sized, const ID: u64> Work<T, ID> { /// } /// ``` /// -/// Note that since the `Work` type is annotated with an id, you can have several `work_struct` +/// Note that since the [`Work`] type is annotated with an id, you can have several `work_struct` /// fields by using a different id for each one. /// /// # Safety /// -/// The [`OFFSET`] constant must be the offset of a field in Self of type [`Work<T, ID>`]. The methods on -/// this trait must have exactly the behavior that the definitions given below have. +/// The [`OFFSET`] constant must be the offset of a field in `Self` of type [`Work<T, ID>`]. The +/// methods on this trait must have exactly the behavior that the definitions given below have. /// -/// [`Work<T, ID>`]: Work /// [`impl_has_work!`]: crate::impl_has_work /// [`OFFSET`]: HasWork::OFFSET pub unsafe trait HasWork<T, const ID: u64 = 0> { /// The offset of the [`Work<T, ID>`] field. - /// - /// [`Work<T, ID>`]: Work const OFFSET: usize; /// Returns the offset of the [`Work<T, ID>`] field. /// - /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not Sized. + /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not + /// [`Sized`]. /// - /// [`Work<T, ID>`]: Work /// [`OFFSET`]: HasWork::OFFSET #[inline] fn get_work_offset(&self) -> usize { @@ -442,8 +452,6 @@ pub unsafe trait HasWork<T, const ID: u64 = 0> { /// # Safety /// /// The provided pointer must point at a valid struct of type `Self`. - /// - /// [`Work<T, ID>`]: Work #[inline] unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID> { // SAFETY: The caller promises that the pointer is valid. @@ -455,8 +463,6 @@ pub unsafe trait HasWork<T, const ID: u64 = 0> { /// # Safety /// /// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`. - /// - /// [`Work<T, ID>`]: Work #[inline] unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self where @@ -473,9 +479,8 @@ pub unsafe trait HasWork<T, const ID: u64 = 0> { /// # Examples /// /// ``` -/// use kernel::impl_has_work; /// use kernel::sync::Arc; -/// use kernel::workqueue::{self, Work}; +/// use kernel::workqueue::{self, impl_has_work, Work}; /// /// struct MyStruct { /// work_field: Work<MyStruct, 17>, @@ -485,8 +490,6 @@ pub unsafe trait HasWork<T, const ID: u64 = 0> { /// impl HasWork<MyStruct, 17> for MyStruct { self.work_field } /// } /// ``` -/// -/// [`HasWork<T, ID>`]: HasWork #[macro_export] macro_rules! impl_has_work { ($(impl$(<$($implarg:ident),*>)? @@ -509,6 +512,7 @@ macro_rules! impl_has_work { } )*}; } +pub use impl_has_work; impl_has_work! { impl<T> HasWork<Self> for ClosureWork<T> { self.work } |