From de49e7c8d96a5a482665851bb99f83e62264fe46 Mon Sep 17 00:00:00 2001 From: longjin Date: Wed, 4 Nov 2026 01:45:24 +0000 Subject: [PATCH 1/2] fix(mm): recover physical allocations at lock-free transaction boundaries Preserve physical frame and fixed-layout allocation failures as typed errors instead of collapsing them into ordinary ENOMEM. Compose the existing charge and global recovery states in MM, with a fixed physical recovery deadline and explicit task, IRQ and NOIO restrictions. Keep GlobalAlloc nonblocking and keep VA exhaustion and unsupported physical layouts terminal. Carry the error through cache births, CPU buffers, kernel stacks and fault retry tokens. Preserve successful write prefixes and delegated completions; retry only the retained synchronization range after payload publication. Let loop backing failures escape NOIO without losing physical provenance. Add pressure-driven file syscall regression coverage and extend allocation, completion and rollback selftests. The same 1 GiB regression fails with ENOMEM before this change and completes after it, including synchronization, thread lifetimes and full data validation. Verify actual recovery from File writes with hardware breakpoints without adding production diagnostics. Validated with default and mm_selftest kernel builds, RISC-V check, formatting and all-features clippy, focused guest memory/VFS/loop regressions, a low-water SQLite integrity workload and OpenCode initial/resume smoke tests. Signed-off-by: longjin --- kernel/src/arch/riscv64/mm/fault.rs | 9 +- kernel/src/arch/x86_64/mm/fault.rs | 5 +- kernel/src/driver/base/block/gendisk/cache.rs | 14 +-- kernel/src/driver/base/block/gendisk/mod.rs | 7 +- kernel/src/driver/block/loop_device/io.rs | 52 +++++++- kernel/src/filesystem/epoll/event_poll.rs | 2 +- kernel/src/filesystem/overlayfs/file.rs | 5 +- kernel/src/filesystem/overlayfs/metadata.rs | 5 +- kernel/src/filesystem/page_cache.rs | 26 ++-- .../filesystem/page_cache/ordered_index.rs | 3 +- kernel/src/filesystem/vfs/file.rs | 27 +++-- kernel/src/filesystem/vfs/file/allocation.rs | 2 +- kernel/src/filesystem/vfs/file/io.rs | 9 +- kernel/src/filesystem/vfs/file/selftest.rs | 21 +++- kernel/src/filesystem/vfs/open.rs | 4 +- .../filesystem/vfs/syscall/sys_fallocate.rs | 2 +- .../src/filesystem/vfs/syscall/sys_ioctl.rs | 2 +- .../vfs/syscall/sys_memfd_create.rs | 2 +- .../filesystem/vfs/syscall/sys_truncate.rs | 2 +- kernel/src/init/initram.rs | 5 +- kernel/src/ipc/syscall/sys_shmat.rs | 2 +- kernel/src/ipc/syscall/sys_shmget.rs | 2 +- kernel/src/libs/futex/futex.rs | 13 +- kernel/src/mm/allocation.rs | 96 ++++++++++++++- kernel/src/mm/allocator/kernel_allocator.rs | 7 ++ kernel/src/mm/cpu_buffer.rs | 7 +- kernel/src/mm/fault.rs | 7 +- kernel/src/mm/fault_recovery.rs | 3 +- kernel/src/mm/kernel_buffer.rs | 9 +- kernel/src/mm/kernel_buffer/selftest.rs | 29 ++--- kernel/src/mm/page.rs | 49 ++++++-- .../src/mm/page/migration/selftest/shared.rs | 6 +- kernel/src/mm/remote_access.rs | 14 ++- kernel/src/mm/syscall/sys_fadvise64.rs | 5 +- kernel/src/mm/syscall/sys_mlock.rs | 2 +- kernel/src/mm/syscall/sys_mlock2.rs | 2 +- kernel/src/mm/syscall/sys_munlock.rs | 2 +- kernel/src/mm/ucontext/address_space.rs | 35 ++++-- kernel/src/mm/ucontext/uprobe/reconcile.rs | 3 +- kernel/src/mm/ucontext/uprobe/site.rs | 2 +- kernel/src/mm/ucontext/vma_ops.rs | 5 +- kernel/src/mm/vmalloc.rs | 20 ++-- kernel/src/mm/vmalloc/selftest.rs | 2 +- kernel/src/net/socket/inet/datagram/mod.rs | 6 +- kernel/src/net/socket/inet/raw/socket.rs | 4 +- .../src/net/socket/inet/stream/lifecycle.rs | 4 +- .../src/net/socket/inet/stream/stream_core.rs | 8 +- kernel/src/net/socket/unix/datagram/mod.rs | 2 +- kernel/src/net/socket/unix/mod.rs | 22 +--- kernel/src/net/socket/unix/stream/mod.rs | 2 +- kernel/src/net/syscall/mod.rs | 6 +- kernel/src/net/syscall/sys_socketpair.rs | 2 +- kernel/src/process/exec.rs | 4 +- kernel/src/process/fork.rs | 2 +- kernel/src/process/kstack.rs | 12 +- kernel/src/process/namespace/setns.rs | 2 +- kernel/src/process/namespace/unshare.rs | 2 +- kernel/src/process/pid_selftest.rs | 2 +- kernel/src/process/posix_timer/cpu.rs | 2 +- kernel/src/process/syscall/sys_pidfdopen.rs | 2 +- kernel/src/process/usermodehelper.rs | 2 +- kernel/src/syscall/user_access.rs | 15 ++- kernel/src/time/cpu_events/mod.rs | 8 +- .../dunitest/suites/normal/global_reclaim.cc | 113 ++++++++++++++++-- 64 files changed, 533 insertions(+), 215 deletions(-) diff --git a/kernel/src/arch/riscv64/mm/fault.rs b/kernel/src/arch/riscv64/mm/fault.rs index 094f7a8de..a13a1abde 100644 --- a/kernel/src/arch/riscv64/mm/fault.rs +++ b/kernel/src/arch/riscv64/mm/fault.rs @@ -1,6 +1,6 @@ +use crate::mm::allocation::AllocationRecoveryState; use crate::{ arch::ipc::signal::Signal, - cgroup::memory::runtime::ChargeRecoveryState, ipc::signal_types::{BUS_ADRERR, SEGV_ACCERR, SEGV_MAPERR}, mm::{ allocation::MemoryAllocationError, @@ -72,7 +72,7 @@ impl RiscV64MMArch { } _ => unreachable!("not a user page fault"), }; - let mut charge_recovery = ChargeRecoveryState::new(); + let mut charge_recovery = AllocationRecoveryState::new(); let mut global_recovery = GlobalFaultRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); @@ -122,7 +122,8 @@ impl RiscV64MMArch { Err(MemoryAllocationError::System( SystemError::EINTR | SystemError::ERESTARTSYS, )) => return Ok(()), - Err(MemoryAllocationError::System(SystemError::ENOMEM)) => { + Err(MemoryAllocationError::System(SystemError::ENOMEM)) + | Err(MemoryAllocationError::Physical) => { reason = VmFaultReason::VM_FAULT_OOM } Err(MemoryAllocationError::System(_)) => { @@ -179,7 +180,7 @@ impl RiscV64MMArch { /// recovery yields to pending signals and a later hardware fault, as on x86. fn recover_charge( failure: crate::cgroup::memory::accounting::ChargeFailure, - state: &mut ChargeRecoveryState, + state: &mut AllocationRecoveryState, account: &crate::cgroup::memory::MemoryAccount, flags: FaultFlags, ) -> bool { diff --git a/kernel/src/arch/x86_64/mm/fault.rs b/kernel/src/arch/x86_64/mm/fault.rs index 503f94072..7f1e2f80d 100644 --- a/kernel/src/arch/x86_64/mm/fault.rs +++ b/kernel/src/arch/x86_64/mm/fault.rs @@ -419,7 +419,7 @@ impl X86_64MMArch { }; let mut global_recovery = crate::mm::fault_recovery::GlobalFaultRecoveryState::new(); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); 'fault_retry: loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); @@ -717,7 +717,8 @@ impl X86_64MMArch { } Err(crate::mm::allocation::MemoryAllocationError::System( SystemError::ENOMEM, - )) => { + )) + | Err(crate::mm::allocation::MemoryAllocationError::Physical) => { fault = VmFaultReason::VM_FAULT_OOM; crate::mm::fault::account_fault_result(&data_account, flags, fault); space_guard = current_address_space.write(); diff --git a/kernel/src/driver/base/block/gendisk/cache.rs b/kernel/src/driver/base/block/gendisk/cache.rs index acc21d481..28781c6fa 100644 --- a/kernel/src/driver/base/block/gendisk/cache.rs +++ b/kernel/src/driver/base/block/gendisk/cache.rs @@ -52,10 +52,9 @@ where }; let confirmed_mask = (((1u16 << confirmed_last) - 1) ^ ((1u16 << first) - 1)) as u8; *remaining &= !confirmed_mask; - result.sync_result.map_err(|error| match error { - MemoryAllocationError::System(error) => error, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + result + .sync_result + .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?; if result.written_len != end - start { return Err(SystemError::EIO); } @@ -149,10 +148,9 @@ impl PageCacheBackend for RawBlockBackend { // A completed result belongs to the lower owner's exact-sync protocol // (loop's write_piece already performs that recovery); never replay it. let result = File::recover_io(|| disk.write_payload(offset, bytes, WriteSyncIntent::None))?; - result.sync_result.map_err(|error| match error { - MemoryAllocationError::System(error) => error, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + result + .sync_result + .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?; Ok(result.written_len) } diff --git a/kernel/src/driver/base/block/gendisk/mod.rs b/kernel/src/driver/base/block/gendisk/mod.rs index a89597421..ac2ca5d01 100644 --- a/kernel/src/driver/base/block/gendisk/mod.rs +++ b/kernel/src/driver/base/block/gendisk/mod.rs @@ -280,10 +280,9 @@ impl GenDisk { })?; // The lower owner has already submitted this prefix and owns exact // sync recovery. Its completion error must never replay the payload. - result.sync_result.map_err(|error| match error { - crate::mm::allocation::MemoryAllocationError::System(error) => error, - crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + result + .sync_result + .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?; Ok(result.written_len) } diff --git a/kernel/src/driver/block/loop_device/io.rs b/kernel/src/driver/block/loop_device/io.rs index 74b49cde1..6ef460f61 100644 --- a/kernel/src/driver/block/loop_device/io.rs +++ b/kernel/src/driver/block/loop_device/io.rs @@ -168,8 +168,19 @@ impl LoopDevice { }) .map_err(Self::backing_error)?; result.sync_result = Self::normalize_backing_sync_result(result.sync_result); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); while let Err(error) = core::mem::replace(&mut result.sync_result, Ok(())) { + if matches!(error, MemoryAllocationError::Physical) { + // The loop pin/gate and NOIO scope still belong to our caller. + // Let its lock-free boundary finish the retained sync range; + // never consume the deferred completions or replay payload. + result.sync_result = Err(if result.sync_range.is_some() { + error + } else { + error.into_errno().into() + }); + break; + } // Payload is committed: recover only its retained exact sync range. if let Err(error) = error.recover(&mut recovery) { result.sync_result = Err(error.into()); @@ -254,7 +265,12 @@ impl LoopDevice { Ok(result) => result, // This is a data error, not a recoverable sync failure. No // invented sync_range may replay an unsubmitted remainder. - Err(error) if done != 0 => return Ok(DelegatedWriteResult::new(done, Err(error))), + Err(error) if done != 0 => { + return Ok(DelegatedWriteResult::new( + done, + Err(error.into_errno().into()), + )) + } Err(error) => return Err(error), }; let expected = if full { count } else { LBA_SIZE }; @@ -317,15 +333,35 @@ impl LoopDevice { fn recover_backing_io( mut attempt: impl FnMut() -> Result, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); loop { match attempt() { Ok(value) => return Ok(value), + // Every caller retains loop admission and a NOIO scope. A + // physical failure must escape with its provenance intact so + // the outer transaction can reclaim after releasing them. + Err(error @ MemoryAllocationError::Physical) => return Err(error), Err(error) => error.recover(&mut recovery)?, } } } + #[cfg(feature = "mm_selftest")] + pub(crate) fn physical_boundary_selftest() -> bool { + let _noio = crate::cgroup::memory::io::MemoryIoScope::enter_noio(); + let mut attempts = 0; + let result: Result<(), MemoryAllocationError> = Self::recover_backing_io(|| { + attempts += 1; + Err(MemoryAllocationError::Physical) + }); + attempts == 1 + && matches!(result, Err(MemoryAllocationError::Physical)) + && matches!( + Self::normalize_backing_sync_result(Err(MemoryAllocationError::Physical)), + Err(MemoryAllocationError::Physical) + ) + } + fn normalize_backing_sync_result( result: Result<(), MemoryAllocationError>, ) -> Result<(), MemoryAllocationError> { @@ -333,7 +369,9 @@ impl LoopDevice { // it to BLK_STS_IOERR. No backing flush error is a successful flush. match result { Ok(()) => Ok(()), - Err(error @ MemoryAllocationError::Charge(_)) => Err(error), + Err(error @ (MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical)) => { + Err(error) + } Err(MemoryAllocationError::System(_)) => Err(SystemError::EIO.into()), } } @@ -361,14 +399,16 @@ impl LoopDevice { | SystemError::ETIME => error.into(), _ => SystemError::EIO.into(), }, - error @ MemoryAllocationError::Charge(_) => error, + error @ (MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => error, } } pub(super) fn legacy_io_terminal(error: MemoryAllocationError) -> SystemError { match error { MemoryAllocationError::System(error) => error, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, + MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical => { + SystemError::ENOMEM + } } } } diff --git a/kernel/src/filesystem/epoll/event_poll.rs b/kernel/src/filesystem/epoll/event_poll.rs index 4ca03428c..17c11184a 100644 --- a/kernel/src/filesystem/epoll/event_poll.rs +++ b/kernel/src/filesystem/epoll/event_poll.rs @@ -297,7 +297,7 @@ impl EventPoll { if !dst_file.supports_poll() { return Err(SystemError::ENOSYS); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let (ready_state, poll_epitems) = { let guard = if nonblock { diff --git a/kernel/src/filesystem/overlayfs/file.rs b/kernel/src/filesystem/overlayfs/file.rs index e91680124..d4e5b61ed 100644 --- a/kernel/src/filesystem/overlayfs/file.rs +++ b/kernel/src/filesystem/overlayfs/file.rs @@ -128,10 +128,7 @@ pub(super) fn write_at( // The legacy scalar inode ABI cannot retain a completed write while // unwinding for recovery. A backing sync charge is a terminal failure; // returning it as recoverable would repeat the already committed data. - result.sync_result.map_err(|error| match error { - crate::mm::allocation::MemoryAllocationError::System(error) => error, - crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + result.sync_result.map_err(|error| error.into_errno())?; Ok(result.written_len) } diff --git a/kernel/src/filesystem/overlayfs/metadata.rs b/kernel/src/filesystem/overlayfs/metadata.rs index 7af397948..67e356d14 100644 --- a/kernel/src/filesystem/overlayfs/metadata.rs +++ b/kernel/src/filesystem/overlayfs/metadata.rs @@ -349,10 +349,7 @@ impl XattrBuffer { /// backing errno, and report rejected scratch admission without reclaim/OOM /// under the caller's resource locks or overridden backing credentials. pub(super) fn xattr_read_error(error: MemoryAllocationError) -> SystemError { - match error { - MemoryAllocationError::System(error) => error, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - } + error.into_errno() } fn for_each_xattr_name>( diff --git a/kernel/src/filesystem/page_cache.rs b/kernel/src/filesystem/page_cache.rs index 3fd4664c4..9c98c9414 100644 --- a/kernel/src/filesystem/page_cache.rs +++ b/kernel/src/filesystem/page_cache.rs @@ -2081,7 +2081,7 @@ impl PageEntry { accounted_unevictable: AtomicBool::new(false), active_users: AtomicUsize::new(0), }) - .map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM))?; + .map_err(|_| MemoryAllocationError::Physical)?; Ok(Self { storage }) } @@ -2635,6 +2635,8 @@ impl PageCache { let mut pages = AccountedPageMap::new_in(HeapPlanAllocator::Auto); pages .try_reserve(required) + // Auto backing may fail its virtual-address reservation. Preserve + // that opaque allocation failure rather than guessing physical. .map_err(|_| SystemError::ENOMEM)?; let (pointer, layout) = pages.raw_table().allocation_info(); let charge = match unsafe { @@ -2658,9 +2660,12 @@ impl PageCache { HeapPlan::Physical.footprint(layout), )?; pages = AccountedPageMap::new_in(HeapPlanAllocator::Physical); - pages - .try_reserve(required) - .map_err(|_| SystemError::ENOMEM)?; + pages.try_reserve(required).map_err(|error| match error { + hashbrown::TryReserveError::CapacityOverflow => SystemError::ENOMEM.into(), + hashbrown::TryReserveError::AllocError { .. } => { + MemoryAllocationError::Physical + } + })?; let (direct_pointer, actual_layout) = pages.raw_table().allocation_info(); assert_eq!( actual_layout, layout, @@ -3196,10 +3201,7 @@ impl PageCache { } fn legacy_allocation_error(error: MemoryAllocationError) -> SystemError { - match error { - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - MemoryAllocationError::System(error) => error, - } + error.into_errno() } fn allocate_page_with_owner( @@ -3227,13 +3229,12 @@ impl PageCache { charge, ) } else { - crate::mm::page::allocate_registered_data_page( + crate::mm::page::allocate_registered_data_page_typed( page_type, self.page_flags(), Some(charge), ) } - .map_err(MemoryAllocationError::System) } fn prepare_entry( @@ -3242,9 +3243,8 @@ impl PageCache { state: PageState, creator: &Arc, ) -> Result, MemoryAllocationError> { - let result = PageEntry::try_new(page.clone(), state, creator).and_then(|entry| { - Arc::try_new(entry).map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM)) - }); + let result = PageEntry::try_new(page.clone(), state, creator) + .and_then(|entry| Arc::try_new(entry).map_err(|_| MemoryAllocationError::Physical)); if result.is_err() { self.discard_unlinked_page(page); } diff --git a/kernel/src/filesystem/page_cache/ordered_index.rs b/kernel/src/filesystem/page_cache/ordered_index.rs index cfe9c9929..99236e676 100644 --- a/kernel/src/filesystem/page_cache/ordered_index.rs +++ b/kernel/src/filesystem/page_cache/ordered_index.rs @@ -5,7 +5,6 @@ use crate::cgroup::memory::{accounting::KernelByteKind, object::ObjectAccount, o use crate::mm::allocation::MemoryAllocationError; use alloc::sync::Arc; use core::ops::{Bound, RangeBounds}; -use system_error::SystemError; #[derive(Debug)] pub(super) struct Node { @@ -27,7 +26,7 @@ impl Node { right: None, height: 1, }) - .map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM)) + .map_err(|_| MemoryAllocationError::Physical) } fn height(node: &Option>) -> i32 { diff --git a/kernel/src/filesystem/vfs/file.rs b/kernel/src/filesystem/vfs/file.rs index fb5eaf533..a884056b4 100644 --- a/kernel/src/filesystem/vfs/file.rs +++ b/kernel/src/filesystem/vfs/file.rs @@ -1070,9 +1070,12 @@ impl File { ) -> Result<(), MemoryAllocationError> { let inode = self.inode(); let sync_result = inode.sync_file_range(start, end, datasync, self.private_data.lock()); - // Charge is an internal retry, not the final fsync result. Preserve + // Recoverable allocation failure is not the final fsync result. Preserve // the OFD cursor until synchronization reaches a terminal result. - if matches!(&sync_result, Err(MemoryAllocationError::Charge(_))) { + if sync_result + .as_ref() + .is_err_and(|error| error.is_recoverable()) + { return sync_result; } let wb_result = match inode.page_cache() { @@ -1313,12 +1316,10 @@ impl File { } Ok(false) => {} Err(error) => { - // Data is already committed. Never let a metadata Charge - // escape to the primary-write recovery loop and replay it. - result.sync_result = Err(match error { - MemoryAllocationError::Charge(_) => SystemError::ENOMEM.into(), - error => error, - }); + // Data is already committed and there is no continuation + // for this metadata operation. A sync retry cannot finish + // it, and a primary-write retry would duplicate the data. + result.sync_result = Err(error.into_errno().into()); return; } } @@ -2607,7 +2608,7 @@ impl File { result.completion = None; result.deferred.clear(); } - Err(MemoryAllocationError::Charge(_)) => { + Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => { if let Some(completion) = result.completion.take() { if result.deferred.try_reserve(1).is_err() { result.sync_result = Err(SystemError::ENOMEM.into()); @@ -2740,7 +2741,9 @@ impl File { Ok(_) => {} // Speculative prefetch never kills a task for an optional page. // Required read_at admission is handled by the inode outer. - Err(crate::mm::allocation::MemoryAllocationError::Charge(_)) => return Ok(()), + Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => { + return Ok(()) + } Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { return Err(error) } @@ -2760,7 +2763,9 @@ impl File { &creator, ) { Ok(_) => {} - Err(crate::mm::allocation::MemoryAllocationError::Charge(_)) => return Ok(()), + Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => { + return Ok(()) + } Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { return Err(error) } diff --git a/kernel/src/filesystem/vfs/file/allocation.rs b/kernel/src/filesystem/vfs/file/allocation.rs index f0b0fcece..691e06306 100644 --- a/kernel/src/filesystem/vfs/file/allocation.rs +++ b/kernel/src/filesystem/vfs/file/allocation.rs @@ -17,7 +17,7 @@ impl PreparedFile { /// Guardless storage-only recovery. Call before instance reservations, /// inode open side effects and publication; this does not open a File. pub(crate) fn new_recovering() -> Result { - let mut state = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut state = crate::mm::allocation::AllocationRecoveryState::default(); loop { match Self::try_new() { Ok(storage) => return Ok(storage), diff --git a/kernel/src/filesystem/vfs/file/io.rs b/kernel/src/filesystem/vfs/file/io.rs index 5afdf12dc..fa908837d 100644 --- a/kernel/src/filesystem/vfs/file/io.rs +++ b/kernel/src/filesystem/vfs/file/io.rs @@ -2,7 +2,7 @@ //! use the typed single-attempt methods instead, preserving their transaction. use super::{DelegatedWriteResult, File, MemoryAllocationError, SystemError}; -use crate::cgroup::memory::runtime::ChargeRecoveryState; +use crate::mm::allocation::AllocationRecoveryState; impl File { pub(super) fn set_completed_sync_range( @@ -53,7 +53,7 @@ impl File { pub(crate) fn recover_io( mut attempt: impl FnMut() -> Result, ) -> Result { - let mut recovery = ChargeRecoveryState::default(); + let mut recovery = AllocationRecoveryState::default(); loop { match attempt() { Ok(value) => return Ok(value), @@ -68,7 +68,7 @@ impl File { &self, mut result: DelegatedWriteResult, ) -> Result { - let mut recovery = ChargeRecoveryState::default(); + let mut recovery = AllocationRecoveryState::default(); loop { match core::mem::replace(&mut result.sync_result, Ok(())) { Ok(()) => { @@ -95,8 +95,7 @@ impl File { self.complete_write_success(&mut result); Ok(result.written_len) } - Err(MemoryAllocationError::System(error)) => Err(error), - Err(MemoryAllocationError::Charge(_)) => Err(SystemError::ENOMEM), + Err(error) => Err(error.into_errno()), } } diff --git a/kernel/src/filesystem/vfs/file/selftest.rs b/kernel/src/filesystem/vfs/file/selftest.rs index aaf90ecf2..923ab6295 100644 --- a/kernel/src/filesystem/vfs/file/selftest.rs +++ b/kernel/src/filesystem/vfs/file/selftest.rs @@ -152,6 +152,7 @@ struct SyncProbeInode { syncs: AtomicUsize, valid: AtomicBool, terminal_error: bool, + physical_error: bool, synchronous_after_write: bool, } @@ -222,6 +223,9 @@ impl IndexNode for SyncProbeInode { ); } if attempt == 0 { + if self.physical_error { + return Err(MemoryAllocationError::Physical); + } // Generate an actual typed quota failure on an isolated account. // Restore its margin before returning: real recovery can then // retry safely without reclaiming or selecting any OOM victim. @@ -276,12 +280,14 @@ impl IndexNode for SyncProbeInode { } } -/// Post-commit Charge retries synchronization only, preserving the strongest +/// Post-commit Charge/Physical retries synchronization only, preserving the strongest /// intent, translated exact range, unpublished position and typed errseq. /// The owning deferred completions must also release every retained File. pub(crate) fn run_write_completion() -> Result { - let mut valid = true; - for (terminal_error, mapping_error, full_sync, synchronous_after_write) in [ + let mut valid = !MemoryAllocationError::System(SystemError::ENOMEM).is_recoverable() + && MemoryAllocationError::Physical.is_recoverable() + && MemoryAllocationError::Physical.into_errno() == SystemError::ENOMEM; + for (physical_error, (terminal_error, mapping_error, full_sync, synchronous_after_write)) in [ (false, false, true, false), (true, false, true, false), (false, true, true, false), @@ -289,7 +295,10 @@ pub(crate) fn run_write_completion() -> Result { // Neither OFD requests sync initially. Only the leaf's post-write // mount-policy resample can discover Data and retain its retry range. (false, false, false, true), - ] { + ] + .into_iter() + .flat_map(|case| [false, true].map(|physical| (physical, case))) + { // Root accounts intentionally bypass controller limits. Use an // isolated child so max=0 really produces a recoverable quota error. let root = MemoryAccount::new(0x776602, None)?; @@ -306,6 +315,7 @@ pub(crate) fn run_write_completion() -> Result { syncs: AtomicUsize::new(0), valid: AtomicBool::new(true), terminal_error, + physical_error, synchronous_after_write, }) .map_err(|_| SystemError::ENOMEM)?; @@ -333,6 +343,7 @@ pub(crate) fn run_write_completion() -> Result { syncs: AtomicUsize::new(0), valid: AtomicBool::new(true), terminal_error: false, + physical_error: false, synchronous_after_write: false, }) .map_err(|_| SystemError::ENOMEM)?; @@ -370,7 +381,7 @@ pub(crate) fn run_write_completion() -> Result { && root.current_pages() == 0; if !outcome_valid || !progress_valid { log::error!( - "File completion selftest: terminal={terminal_error} mapping={mapping_error} \ + "File completion selftest: physical={physical_error} terminal={terminal_error} mapping={mapping_error} \ full={full_sync} resample={synchronous_after_write} outcome={outcome:?} \ pos={} writes={}/{} syncs={}/{} ranges={}/{}", outer.pos(), diff --git a/kernel/src/filesystem/vfs/open.rs b/kernel/src/filesystem/vfs/open.rs index 635093885..ec4a376d1 100644 --- a/kernel/src/filesystem/vfs/open.rs +++ b/kernel/src/filesystem/vfs/open.rs @@ -422,7 +422,7 @@ pub(crate) fn do_sys_openat2(dirfd: i32, path: &str, how: OpenHow) -> Result break storage, @@ -870,7 +870,7 @@ pub fn do_open_execat_with_flags( super::permission::check_inode_permission(&inode, &metadata, PermissionMask::MAY_EXEC)?; // 创建File对象，使用O_RDONLY | O_CLOEXEC - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); let file = loop { match File::new(inode.clone(), FileFlags::O_RDONLY | FileFlags::O_CLOEXEC) { Ok(file) => break file, diff --git a/kernel/src/filesystem/vfs/syscall/sys_fallocate.rs b/kernel/src/filesystem/vfs/syscall/sys_fallocate.rs index e71c596ff..dd56c57b4 100644 --- a/kernel/src/filesystem/vfs/syscall/sys_fallocate.rs +++ b/kernel/src/filesystem/vfs/syscall/sys_fallocate.rs @@ -51,7 +51,7 @@ impl Syscall for SysFallocateHandle { if let Some(file) = fd_table_guard.get_file_by_fd(fd) { drop(fd_table_guard); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); loop { match crate::filesystem::vfs::vcore::vfs_fallocate_file( file.clone(), diff --git a/kernel/src/filesystem/vfs/syscall/sys_ioctl.rs b/kernel/src/filesystem/vfs/syscall/sys_ioctl.rs index d4c47a457..c943f3d96 100644 --- a/kernel/src/filesystem/vfs/syscall/sys_ioctl.rs +++ b/kernel/src/filesystem/vfs/syscall/sys_ioctl.rs @@ -89,7 +89,7 @@ impl Syscall for SysIoctlHandle { } _ => { // 其他命令转发给inode处理 - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); loop { // The single attempt owns the private guard. It is gone // before recovery, and drivers admit resources before diff --git a/kernel/src/filesystem/vfs/syscall/sys_memfd_create.rs b/kernel/src/filesystem/vfs/syscall/sys_memfd_create.rs index 5e21abdb2..3d7d0bdf8 100644 --- a/kernel/src/filesystem/vfs/syscall/sys_memfd_create.rs +++ b/kernel/src/filesystem/vfs/syscall/sys_memfd_create.rs @@ -62,7 +62,7 @@ impl Syscall for SysMemfdCreate { return Err(SystemError::ENOSYS); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let file = loop { let mut attempt_name = Vec::new(); attempt_name diff --git a/kernel/src/filesystem/vfs/syscall/sys_truncate.rs b/kernel/src/filesystem/vfs/syscall/sys_truncate.rs index 4e57aed57..118a55d57 100644 --- a/kernel/src/filesystem/vfs/syscall/sys_truncate.rs +++ b/kernel/src/filesystem/vfs/syscall/sys_truncate.rs @@ -69,7 +69,7 @@ impl Syscall for SysTruncateHandle { } } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); while let Err(error) = vfs_truncate(target.clone(), length) { error.recover(&mut recovery)?; } diff --git a/kernel/src/init/initram.rs b/kernel/src/init/initram.rs index e0b0b2a20..7e36ce16d 100644 --- a/kernel/src/init/initram.rs +++ b/kernel/src/init/initram.rs @@ -97,10 +97,7 @@ struct CpioEntryInfo { pub fn initramfs_init() -> Result<(), SystemError> { // The initcall ABI is a terminal bootstrap boundary, not a task-context // reclaim/retry site. Keep admission errors typed throughout extraction. - initramfs_init_typed().map_err(|error| match error { - crate::mm::allocation::MemoryAllocationError::System(error) => error, - crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - }) + initramfs_init_typed().map_err(|error| error.into_errno()) } fn initramfs_init_typed() -> Result<(), crate::mm::allocation::MemoryAllocationError> { diff --git a/kernel/src/ipc/syscall/sys_shmat.rs b/kernel/src/ipc/syscall/sys_shmat.rs index 7f0e45e03..98346d59e 100644 --- a/kernel/src/ipc/syscall/sys_shmat.rs +++ b/kernel/src/ipc/syscall/sys_shmat.rs @@ -79,7 +79,7 @@ pub(super) fn do_kernel_shmat( let readonly = shmflg.contains(ShmFlags::SHM_RDONLY); // attach_guard pins the exact segment; attach_begin's manager guard was // released above. Retrying private File birth must not relookup a reused ID. - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let sysv_attach = loop { match attach_guard.create_attach(readonly) { Ok(attach) => break attach, diff --git a/kernel/src/ipc/syscall/sys_shmget.rs b/kernel/src/ipc/syscall/sys_shmget.rs index 9b8a9a848..2255b754d 100644 --- a/kernel/src/ipc/syscall/sys_shmget.rs +++ b/kernel/src/ipc/syscall/sys_shmget.rs @@ -54,7 +54,7 @@ pub(super) fn do_kernel_shmget( } let ipcns = ProcessManager::current_ipcns(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); match key { IPC_PRIVATE => { diff --git a/kernel/src/libs/futex/futex.rs b/kernel/src/libs/futex/futex.rs index 4a41566a0..ed0fcdea8 100644 --- a/kernel/src/libs/futex/futex.rs +++ b/kernel/src/libs/futex/futex.rs @@ -395,7 +395,7 @@ impl Futex { ); let current_address_space = AddressSpace::current()?; let mut flags = FaultFlags::FAULT_FLAG_WRITE; - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); @@ -459,6 +459,17 @@ impl Futex { )?; continue; } + Err(error @ crate::mm::allocation::MemoryAllocationError::Physical) => { + if let Err(error) = error.recover(&mut charge_recovery) { + crate::mm::fault::account_fault_recovery_failure( + &data_account, + flags, + error.clone(), + ); + return Err(error); + } + continue; + } Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { Err(error) } diff --git a/kernel/src/mm/allocation.rs b/kernel/src/mm/allocation.rs index edf0fd023..c1f7fe82e 100644 --- a/kernel/src/mm/allocation.rs +++ b/kernel/src/mm/allocation.rs @@ -10,6 +10,8 @@ use crate::mm::oom_reaper::JobBirthError; #[derive(Debug)] pub(crate) enum MemoryAllocationError { + /// A real frame or fixed-layout heap allocation failed. Never infer this from errno. + Physical, Charge(ChargeFailure), System(SystemError), } @@ -30,17 +32,75 @@ impl From for MemoryAllocationError { fn from(error: JobBirthError) -> Self { match error { JobBirthError::Charge(failure) => Self::Charge(failure), - JobBirthError::Physical(_) => Self::System(SystemError::ENOMEM), + JobBirthError::Physical(_) => Self::Physical, } } } +/// One resource-lock-free outer transaction. Quota and global pressure have +/// distinct accounting and budgets; unrelated syscalls never share this state. +pub(crate) struct AllocationRecoveryState { + charge: ChargeRecoveryState, + global: super::fault_recovery::GlobalFaultRecoveryState, + physical_deadline: Option, +} + +impl AllocationRecoveryState { + pub(crate) const fn new() -> Self { + Self { + charge: ChargeRecoveryState::new(), + global: super::fault_recovery::GlobalFaultRecoveryState::new(), + physical_deadline: None, + } + } + + fn recover_physical(&mut self) -> Result<(), SystemError> { + // Check task/IRQ/preemption before reading any task-local NOIO state. + if !super::vmalloc::task_context() || crate::cgroup::memory::io::noio_active() { + return Err(SystemError::ENOMEM); + } + let deadline = *self + .physical_deadline + .get_or_insert_with(|| crate::time::timer::next_n_us_timer_jiffies(10_000_000)); + if crate::time::timer::clock() >= deadline { + return Err(SystemError::ENOMEM); + } + // Reuse fault reclaim's bounded scan and killable writeback waits, but + // keep a fixed transaction deadline even when unrelated pages release. + let retry = self.global.retry()?; + if retry && crate::time::timer::clock() < deadline { + Ok(()) + } else { + Err(SystemError::ENOMEM) + } + } +} + +impl Default for AllocationRecoveryState { + fn default() -> Self { + Self::new() + } +} + impl MemoryAllocationError { + pub(crate) fn is_recoverable(&self) -> bool { + matches!(self, Self::Charge(_) | Self::Physical) + } + + /// Terminal/legacy boundary; an errno never grants recovery authority. + pub(crate) fn into_errno(self) -> SystemError { + match self { + Self::System(error) => error, + Self::Physical | Self::Charge(_) => SystemError::ENOMEM, + } + } + /// Call only after releasing resource guards and failed MM owners. /// Ok means retry the complete unpublished operation with a fresh creator. - pub(crate) fn recover(self, state: &mut ChargeRecoveryState) -> Result<(), SystemError> { + pub(crate) fn recover(self, state: &mut AllocationRecoveryState) -> Result<(), SystemError> { match self { Self::System(error) => Err(error), + Self::Physical => state.recover_physical(), // The hard counter cap is never reclaimable, including during // bootstrap before a current task exists. Do not query NOIO first. Self::Charge(failure) if failure.reason == ChargeFailureReason::CounterOverflow => { @@ -52,7 +112,7 @@ impl MemoryAllocationError { } else { MemoryAllocPolicy::ReclaimThenOom }; - match recover_charge(&failure, policy, 0, state)? { + match recover_charge(&failure, policy, 0, &mut state.charge)? { RecoveryDecision::RetryOperation => Ok(()), RecoveryDecision::Exhausted => Err(SystemError::ENOMEM), } @@ -60,3 +120,33 @@ impl MemoryAllocationError { } } } + +#[cfg(feature = "mm_selftest")] +pub(crate) fn recovery_selftest() -> bool { + if !super::vmalloc::task_context() { + return false; + } + let mut state = AllocationRecoveryState::new(); + // An errno alone must never enter reclaim or start its budget. + for error in [SystemError::ENOMEM, SystemError::EIO, SystemError::EFAULT] { + if MemoryAllocationError::System(error.clone()).is_recoverable() + || MemoryAllocationError::System(error.clone()).recover(&mut state) != Err(error) + || state.physical_deadline.is_some() + { + return false; + } + } + { + let _noio = crate::cgroup::memory::io::MemoryIoScope::enter_noio(); + if MemoryAllocationError::Physical.recover(&mut state) != Err(SystemError::ENOMEM) + || state.physical_deadline.is_some() + { + return false; + } + } + state.physical_deadline = Some(crate::time::timer::clock()); + MemoryAllocationError::Physical.recover(&mut state) == Err(SystemError::ENOMEM) + && MemoryAllocationError::Physical.is_recoverable() + && MemoryAllocationError::Physical.into_errno() == SystemError::ENOMEM + && crate::driver::block::loop_device::LoopDevice::physical_boundary_selftest() +} diff --git a/kernel/src/mm/allocator/kernel_allocator.rs b/kernel/src/mm/allocator/kernel_allocator.rs index 1826ece5c..c3f909bf0 100644 --- a/kernel/src/mm/allocator/kernel_allocator.rs +++ b/kernel/src/mm/allocator/kernel_allocator.rs @@ -55,6 +55,13 @@ pub(super) fn large_allocation_bytes() -> u64 { } impl KernelAllocator { + /// Structural limit of the direct backend, independent of current pressure. + pub(crate) fn supports_physical_layout(layout: Layout) -> bool { + !super::footprint::uses_buddy(layout) + || super::footprint::buddy_frame_count(layout) + <= (1usize << (super::buddy::BUDDY_ORDER_COUNT - 1)) + } + unsafe fn alloc_in_buddy( &self, layout: Layout, diff --git a/kernel/src/mm/cpu_buffer.rs b/kernel/src/mm/cpu_buffer.rs index ce055f23b..e3ca2b3f0 100644 --- a/kernel/src/mm/cpu_buffer.rs +++ b/kernel/src/mm/cpu_buffer.rs @@ -90,11 +90,8 @@ impl CpuBuffer { #[cfg(feature = "mm_selftest")] retired: None, })?; - let buffer = - KernelBuffer::try_new_zeroed_virtual(capacity).map_err(|error| match error { - super::allocation::MemoryAllocationError::System(error) => error, - super::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + let buffer = KernelBuffer::try_new_zeroed_virtual(capacity) + .map_err(|error| error.into_errno())?; node.value_mut().buffer = Some(buffer); return Ok(Self { backend: Backend::Mapped(Some(node)), diff --git a/kernel/src/mm/fault.rs b/kernel/src/mm/fault.rs index f0c10e0cb..ce149fc9a 100644 --- a/kernel/src/mm/fault.rs +++ b/kernel/src/mm/fault.rs @@ -82,7 +82,7 @@ pub(crate) fn account_fault_result( /// PGFAULT, but an interrupted incomplete fault is not an OOM/minor/major fault. pub(crate) fn recover_fault_charge( failure: ChargeFailure, - state: &mut crate::cgroup::memory::runtime::ChargeRecoveryState, + state: &mut crate::mm::allocation::AllocationRecoveryState, account: &MemoryAccount, flags: FaultFlags, ) -> Result<(), SystemError> { @@ -343,9 +343,8 @@ impl<'a> PageFaultMessage<'a> { crate::mm::allocation::MemoryAllocationError::Charge(failure) => { self.fail_charge(failure) } - crate::mm::allocation::MemoryAllocationError::System(SystemError::ENOMEM) => { - VmFaultReason::VM_FAULT_OOM - } + crate::mm::allocation::MemoryAllocationError::System(SystemError::ENOMEM) + | crate::mm::allocation::MemoryAllocationError::Physical => VmFaultReason::VM_FAULT_OOM, crate::mm::allocation::MemoryAllocationError::System(_) => { VmFaultReason::VM_FAULT_SIGBUS } diff --git a/kernel/src/mm/fault_recovery.rs b/kernel/src/mm/fault_recovery.rs index de1a2edec..bf33477b2 100644 --- a/kernel/src/mm/fault_recovery.rs +++ b/kernel/src/mm/fault_recovery.rs @@ -1,4 +1,5 @@ -//! Physical allocation recovery at the resource-lock-free user fault boundary. +//! Shared physical allocation reclaim at resource-lock-free transaction boundaries. +//! User faults own their OOM handoff; fallible operations own a fixed total deadline. //! Charge/memcg failures retain their separate policy and limiting account. use core::sync::atomic::{AtomicUsize, Ordering}; diff --git a/kernel/src/mm/kernel_buffer.rs b/kernel/src/mm/kernel_buffer.rs index 6177c4bf0..7f28484ce 100644 --- a/kernel/src/mm/kernel_buffer.rs +++ b/kernel/src/mm/kernel_buffer.rs @@ -54,7 +54,7 @@ impl KernelBuffer { /// Prepare local scratch at a resource-lock-free task boundary. Only the /// unpublished allocation is retried; callers must perform I/O afterwards. pub(crate) fn new_zeroed_recovering(len: usize) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); loop { match Self::try_new_zeroed(len) { Ok(buffer) => return Ok(buffer), @@ -188,6 +188,11 @@ impl KernelBuffer { // high-order pressure and rounding waste. Otherwise retain the heap // backend for all sizes, with the same admission and no-retry policy. if heap_supported && (len <= 2 * MMArch::PAGE_SIZE || !virtual_supported) { + if !crate::mm::allocator::kernel_allocator::KernelAllocator::supports_physical_layout( + layout, + ) { + return Err(SystemError::ENOMEM.into()); + } let charge = owner.try_charge_allocation(KernelByteKind::SlabUnreclaimable, layout)?; #[cfg(feature = "mm_selftest")] let pointer = if fail_heap { @@ -210,7 +215,7 @@ impl KernelBuffer { // above is propagated, not converted into a fallback opportunity. drop(charge); if len <= MMArch::PAGE_SIZE || !virtual_supported { - return Err(SystemError::ENOMEM.into()); + return Err(MemoryAllocationError::Physical); } } #[cfg(feature = "mm_selftest")] diff --git a/kernel/src/mm/kernel_buffer/selftest.rs b/kernel/src/mm/kernel_buffer/selftest.rs index 8f32b75d6..65daa1f72 100644 --- a/kernel/src/mm/kernel_buffer/selftest.rs +++ b/kernel/src/mm/kernel_buffer/selftest.rs @@ -121,10 +121,7 @@ fn failure_policy() -> bool { calls += 1; Err(SystemError::ENOMEM.into()) }); - let rejected = matches!( - &result, - Err(MemoryAllocationError::System(SystemError::ENOMEM)) - ); + let rejected = matches!(&result, Err(MemoryAllocationError::Physical)); drop(result); if !rejected || calls != 0 || !empty(&account) { return false; @@ -235,15 +232,20 @@ fn unsupported_virtual() -> bool { return false; } let result = KernelBuffer::allocate_in(len, owner, false, true, &mut unexpected); - if !matches!( - result, - Err(MemoryAllocationError::System(SystemError::ENOMEM)) - ) || !empty(&account) - { + if !matches!(result, Err(MemoryAllocationError::Physical)) || !empty(&account) { return false; } } - // Even a valid Layout too large for buddy must fail without leaking charge. + // A valid Layout can exceed the backend's structural capacity. Reclaim + // cannot satisfy it, and it must never acquire a charge or retry authority. + let supported = + crate::mm::allocator::kernel_allocator::KernelAllocator::supports_physical_layout; + if !supported(Layout::from_size_align(1usize << 30, 1).unwrap()) + || supported(Layout::from_size_align((1usize << 30) + 1, 1).unwrap()) + || supported(Layout::from_size_align(1usize << 32, 1).unwrap()) + { + return false; + } let result = KernelBuffer::allocate_in(1usize << 32, owner, false, false, &mut unexpected); if !matches!( result, @@ -506,6 +508,8 @@ fn typed_failure_rollback() -> bool { return false; }; let owner = prepared.object_account(); + // An injected data-frame ENOMEM represents a real Physical failure, + // unlike a generic errno returned by the virtual backend itself. // Fail every data-frame boundary in a four-page typed slice. Successfully // donated frames must be unmapped/freed before the charge is released. for fail_after in 0..4 { @@ -530,10 +534,7 @@ fn typed_failure_rollback() -> bool { }, ) .map(|storage| KernelSlice::initialize(storage, 1537, 9usize)); - let failed = matches!( - &result, - Err(MemoryAllocationError::System(SystemError::ENOMEM)) - ); + let failed = matches!(&result, Err(MemoryAllocationError::Physical)); drop(result); failed && calls == fail_after + 1 && storage_released(&account) }); diff --git a/kernel/src/mm/page.rs b/kernel/src/mm/page.rs index f0102c755..c0bef25ad 100644 --- a/kernel/src/mm/page.rs +++ b/kernel/src/mm/page.rs @@ -10,6 +10,7 @@ use core::{ use system_error::SystemError; use unified_init::macros::unified_init; +use super::allocation::MemoryAllocationError; use super::page_lru::{PageLru, PreparedPageLruCapacity}; pub(crate) use super::page_rmap::PageMapping; use super::page_rmap::PageVmaSet; @@ -457,7 +458,25 @@ pub(crate) fn allocate_registered_data_page( flags: PageFlags, charge: Option, ) -> Result, SystemError> { - allocate_registered_data_page_initialized(page_type, flags, charge, |_| Ok(())) + allocate_registered_data_page_typed(page_type, flags, charge) + .map_err(MemoryAllocationError::into_errno) +} + +/// Preserve physical pressure until the cache caller reaches its lock-free +/// recovery boundary. Legacy fault callers retain their original errno API. +pub(crate) fn allocate_registered_data_page_typed( + page_type: PageType, + flags: PageFlags, + charge: Option, +) -> Result, MemoryAllocationError> { + let request = data_frame_request(PageFrameCount::ONE, charge.as_ref()); + allocate_registered_data_page_initialized_with_request( + page_type, + flags, + charge, + |_| Ok(()), + request, + ) } /// Copy before publication: the source Arc pins its frame without a registry @@ -494,6 +513,7 @@ pub(crate) fn allocate_registered_data_page_initialized( allocate_registered_data_page_initialized_with_request( page_type, flags, charge, initialize, request, ) + .map_err(MemoryAllocationError::into_errno) } fn reclaim_io_backing( @@ -537,6 +557,7 @@ pub(crate) fn reclaim_io_admission_selftest() -> bool { && !reclaim_io_backing(MemoryKind::File, true, ReserveLevel::OomExit) && scan_visit_budget_selftest() && crate::mm::fault_recovery::budget_selftest() + && crate::mm::allocation::recovery_selftest() } /// Only PageCache's File path may identify a necessary stacked-I/O backing @@ -546,9 +567,9 @@ pub(crate) fn allocate_registered_file_cache_page( page_type: PageType, flags: PageFlags, charge: MemoryCharge, -) -> Result, SystemError> { +) -> Result, MemoryAllocationError> { if !matches!(&page_type, PageType::File(_)) || charge.kind() != MemoryKind::File { - return Err(SystemError::EINVAL); + return Err(SystemError::EINVAL.into()); } let request = if reclaim_io_backing( charge.kind(), @@ -590,12 +611,12 @@ fn allocate_registered_data_page_initialized_with_request( charge: Option, initialize: impl FnOnce(PhysAddr) -> Result<(), SystemError>, request: FrameAllocationRequest, -) -> Result, SystemError> { +) -> Result, MemoryAllocationError> { if charge .as_ref() .is_some_and(|charge| charge.pages() != 1 || !page_type.accepts_memory_kind(charge.kind())) { - return Err(SystemError::EINVAL); + return Err(SystemError::EINVAL.into()); } // Only this registered single-page path has the audited data lifecycle. // Raw/contiguous/legacy-manager callers keep their original default. This @@ -612,7 +633,7 @@ fn allocate_registered_data_page_initialized_with_request( LockedFrameAllocator .allocate_request(request) .map(|(paddr, _)| paddr) - .ok_or(SystemError::ENOMEM)? + .ok_or(MemoryAllocationError::Physical)? }; let mut raw = RawFrameExtent { next: paddr, @@ -623,7 +644,7 @@ fn allocate_registered_data_page_initialized_with_request( initialize(paddr)?; // The constructor takes ownership on success AND Arc allocation failure. raw.remaining = 0; - let page = Page::try_new_with_charge(paddr, page_type, flags, charge)?; + let page = Page::try_new_with_charge_typed(paddr, page_type, flags, charge)?; loop { let mut manager_capacity = prepare_manager_capacity()?; let mut lru_capacity = if flags.contains(PageFlags::PG_LRU) { @@ -695,7 +716,7 @@ fn allocate_registered_data_page_initialized_with_request( drop(manager_capacity); match result { Some(Ok(())) => return Ok(page), - Some(Err(error)) => return Err(error), + Some(Err(error)) => return Err(error.into()), None => {} } } @@ -2079,6 +2100,16 @@ impl Page { flags: PageFlags, memory_charge: Option, ) -> Result, SystemError> { + Self::try_new_with_charge_typed(phys_addr, page_type, flags, memory_charge) + .map_err(MemoryAllocationError::into_errno) + } + + fn try_new_with_charge_typed( + phys_addr: PhysAddr, + page_type: PageType, + flags: PageFlags, + memory_charge: Option, + ) -> Result, MemoryAllocationError> { let inner = InnerPage::new( phys_addr, page_type, @@ -2095,7 +2126,7 @@ impl Page { contiguous_owner: None, memory_charge, }) - .map_err(|_| SystemError::ENOMEM)?; + .map_err(|_| MemoryAllocationError::Physical)?; // The unlocked birth API publishes manager/LRU together. Construction // alone must not expose a page through either registry. Ok(page) diff --git a/kernel/src/mm/page/migration/selftest/shared.rs b/kernel/src/mm/page/migration/selftest/shared.rs index dad768da7..ef3b8b513 100644 --- a/kernel/src/mm/page/migration/selftest/shared.rs +++ b/kernel/src/mm/page/migration/selftest/shared.rs @@ -28,10 +28,8 @@ impl CacheFixture { (None, None) } else { let fs = crate::filesystem::ramfs::RamFS::new(); - let fixture_error = |error| match error { - crate::mm::allocation::MemoryAllocationError::System(error) => error, - crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - }; + let fixture_error = + |error: crate::mm::allocation::MemoryAllocationError| error.into_errno(); let inode = fs .root_inode() .create( diff --git a/kernel/src/mm/remote_access.rs b/kernel/src/mm/remote_access.rs index b89889dee..ba1e72404 100644 --- a/kernel/src/mm/remote_access.rs +++ b/kernel/src/mm/remote_access.rs @@ -434,7 +434,7 @@ fn fault_in_remote_page( force: bool, ) -> Result { let mut tried = false; - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&crate::process::ProcessManager::current_pcb()); @@ -574,6 +574,18 @@ fn fault_in_remote_page( continue; } Err(crate::mm::allocation::MemoryAllocationError::System(error)) => Err(error), + Err(error @ crate::mm::allocation::MemoryAllocationError::Physical) => { + if let Err(error) = error.recover(&mut charge_recovery) { + crate::mm::fault::account_fault_recovery_failure( + &data_account, + retry_flags, + error.clone(), + ); + return Err(error); + } + tried = true; + continue; + } Ok(()) => Ok(()), }; if let Err(error) = result { diff --git a/kernel/src/mm/syscall/sys_fadvise64.rs b/kernel/src/mm/syscall/sys_fadvise64.rs index a05364c14..35736e5d8 100644 --- a/kernel/src/mm/syscall/sys_fadvise64.rs +++ b/kernel/src/mm/syscall/sys_fadvise64.rs @@ -181,7 +181,10 @@ pub(crate) fn generic_fadvise( Ok(_) => file.set_ra_state(ra_state)?, // Linux WILLNEED is advisory: failure to prefetch does not // force OOM or make the subsequent required read succeed. - Err(crate::mm::allocation::MemoryAllocationError::Charge(_)) => {} + Err( + crate::mm::allocation::MemoryAllocationError::Charge(_) + | crate::mm::allocation::MemoryAllocationError::Physical, + ) => {} Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { return Err(error) } diff --git a/kernel/src/mm/syscall/sys_mlock.rs b/kernel/src/mm/syscall/sys_mlock.rs index 8d8977703..f43ed4cd7 100644 --- a/kernel/src/mm/syscall/sys_mlock.rs +++ b/kernel/src/mm/syscall/sys_mlock.rs @@ -64,7 +64,7 @@ pub(super) fn do_mlock( let vm = AddressSpace::current()?; let region = VirtRegion::new(start, len); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&crate::process::ProcessManager::current_pcb()); diff --git a/kernel/src/mm/syscall/sys_mlock2.rs b/kernel/src/mm/syscall/sys_mlock2.rs index 5cd9487e7..7f439ffb5 100644 --- a/kernel/src/mm/syscall/sys_mlock2.rs +++ b/kernel/src/mm/syscall/sys_mlock2.rs @@ -74,7 +74,7 @@ fn do_mlock2(start: VirtAddr, len: usize, flags: usize) -> Result Err(error), + Err(error @ crate::mm::allocation::MemoryAllocationError::Physical) => { + if let Err(error) = error.recover(&mut charge_recovery) { + crate::mm::fault::account_fault_recovery_failure( + &data_account, + retry_flags, + error.clone(), + ); + return Err(error); + } + retried = true; + continue; + } Ok(()) => Ok(()), }; if let Err(error) = result { @@ -546,7 +558,7 @@ impl AddressSpace { user_ns: Arc, dumpable: u8, ) -> Result, SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match Self::new_for_task_with_creator( @@ -689,6 +701,7 @@ impl AddressSpace { ) { Ok(mm) => mm, Err(MemoryAllocationError::System(error)) => return Err(error), + Err(MemoryAllocationError::Physical) => return Err(SystemError::ENOMEM), // An unlimited private fixture must not reject admission. This is // a failed assertion, not production scope-to-errno conversion. Err(MemoryAllocationError::Charge(_)) => return Ok(false), @@ -1329,7 +1342,7 @@ impl AddressSpace { round_to_min: bool, allocate_at_once: bool, ) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); let len = page_align_up(len); if map_flags.contains(MapFlags::MAP_SHARED) { if len == 0 { @@ -1524,7 +1537,7 @@ impl AddressSpace { self: &Arc, args: FileMappingWithFileArgs, ) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); let FileMappingWithFileArgs { file, start_vaddr, @@ -2046,7 +2059,7 @@ impl AddressSpace { page_count: PageFrameCount, ) -> Result<(), SystemError> { let region = VirtRegion::new(start_page.virt_address(), page_count.bytes()); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let mut guard = self.write(); @@ -2103,7 +2116,7 @@ impl AddressSpace { prot_flags: ProtFlags, ) -> Result<(), SystemError> { let region = VirtRegion::new(start_page.virt_address(), page_count.bytes()); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2167,7 +2180,7 @@ impl AddressSpace { behavior: MadvFlags, ) -> Result<(), SystemError> { let region = VirtRegion::new(start_page.virt_address(), page_count.bytes()); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2251,7 +2264,7 @@ impl AddressSpace { mremap_flags: MremapFlags, new_vaddr: VirtAddr, ) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2459,7 +2472,7 @@ impl AddressSpace { } pub fn set_brk_wait(self: &Arc, new_addr: VirtAddr) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2537,7 +2550,7 @@ impl AddressSpace { } pub fn sbrk_wait(self: &Arc, incr: isize) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2608,7 +2621,7 @@ impl AddressSpace { self: &Arc, owner: &Arc, ) -> Result, SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { match self.try_clone_for_task(owner) { Ok(child) => return Ok(child), diff --git a/kernel/src/mm/ucontext/uprobe/reconcile.rs b/kernel/src/mm/ucontext/uprobe/reconcile.rs index 82a5a8703..17411504f 100644 --- a/kernel/src/mm/ucontext/uprobe/reconcile.rs +++ b/kernel/src/mm/ucontext/uprobe/reconcile.rs @@ -538,7 +538,8 @@ fn uprobe_register_best_effort_once( crate::mm::allocation::MemoryAllocationError::System(error), )) => Err(error), Err(LockedRegisterError::Allocation( - crate::mm::allocation::MemoryAllocationError::Charge(_), + crate::mm::allocation::MemoryAllocationError::Charge(_) + | crate::mm::allocation::MemoryAllocationError::Physical, )) => Err(SystemError::EAGAIN_OR_EWOULDBLOCK), } } diff --git a/kernel/src/mm/ucontext/uprobe/site.rs b/kernel/src/mm/ucontext/uprobe/site.rs index f7c473f4c..b3f8f4dab 100644 --- a/kernel/src/mm/ucontext/uprobe/site.rs +++ b/kernel/src/mm/ucontext/uprobe/site.rs @@ -541,7 +541,7 @@ pub(super) fn uprobe_register( consumer: &Arc, expected_mapping: &ExpectedProbeMapping, ) -> Result, SystemError> { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); let result = loop { let creator = crate::cgroup::memory::object_for_task(&crate::process::ProcessManager::current_pcb()); diff --git a/kernel/src/mm/ucontext/vma_ops.rs b/kernel/src/mm/ucontext/vma_ops.rs index 0a64118a8..5024e9651 100644 --- a/kernel/src/mm/ucontext/vma_ops.rs +++ b/kernel/src/mm/ucontext/vma_ops.rs @@ -725,10 +725,7 @@ impl InnerAddressSpace { notifications: rollback_notifications, }); } - terminal_error = Some(match err { - MemoryAllocationError::System(err) => err, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - }); + terminal_error = Some(err.into_errno()); break; } }; diff --git a/kernel/src/mm/vmalloc.rs b/kernel/src/mm/vmalloc.rs index d5f99c09a..1f31992ec 100644 --- a/kernel/src/mm/vmalloc.rs +++ b/kernel/src/mm/vmalloc.rs @@ -79,12 +79,12 @@ pub(crate) fn overlaps(start: VirtAddr, end: VirtAddr) -> bool { start < end && start.data() < BASE + SIZE && end.data() > BASE } -unsafe fn metadata_box(value: T) -> Result, SystemError> { +unsafe fn metadata_box(value: T) -> Result, MemoryAllocationError> { let pointer = KERNEL_ALLOCATOR .alloc_metadata(Layout::new::()) .cast::(); if pointer.is_null() { - return Err(SystemError::ENOMEM); + return Err(MemoryAllocationError::Physical); } pointer.write(value); Ok(Box::from_raw(pointer)) @@ -564,7 +564,7 @@ fn allocate_record( |layout| KERNEL_ALLOCATOR.alloc_metadata(layout), ) } - .map_err(|_| SystemError::ENOMEM)?; + .map_err(|_| MemoryAllocationError::Physical)?; let reservation = { POOL.lock_irqsave().reserve( plan.pages, @@ -615,9 +615,15 @@ fn allocate_record( #[cfg(feature = "mm_selftest")] test.step()?; #[cfg(feature = "mm_selftest")] - let frame = data_frame()?; + let frame = data_frame().map_err(|error| { + if error == SystemError::ENOMEM { + MemoryAllocationError::Physical + } else { + error.into() + } + })?; #[cfg(not(feature = "mm_selftest"))] - let frame = frame_zeroed()?; + let frame = frame_zeroed().map_err(|_| MemoryAllocationError::Physical)?; chunk.frames[index] = frame; chunk.acquired += 1; DATA_PAGES.fetch_add(1, Ordering::Relaxed); @@ -679,7 +685,7 @@ fn map_page( )?; #[cfg(feature = "mm_selftest")] test.step()?; - let frame = frame_zeroed()?; + let frame = frame_zeroed().map_err(|_| MemoryAllocationError::Physical)?; let table = LeafTable { record: None, frame, @@ -692,7 +698,7 @@ fn map_page( KERNEL_ALLOCATOR.alloc_metadata(layout) }) } - .map_err(|_| SystemError::ENOMEM)?, + .map_err(|_| MemoryAllocationError::Physical)?, ); let result; { diff --git a/kernel/src/mm/vmalloc/selftest.rs b/kernel/src/mm/vmalloc/selftest.rs index d29206634..5a722431f 100644 --- a/kernel/src/mm/vmalloc/selftest.rs +++ b/kernel/src/mm/vmalloc/selftest.rs @@ -776,7 +776,7 @@ fn global_header_footprint() -> bool { // Private records contain no mapped resources. This exercises the actual // intrusive queue, including producers arriving between worker pops. fn retirement_fifo() -> bool { - fn record(id: usize) -> Result, SystemError> { + fn record(id: usize) -> Result, MemoryAllocationError> { unsafe { metadata_box(AllocationRecord { start: 0, diff --git a/kernel/src/net/socket/inet/datagram/mod.rs b/kernel/src/net/socket/inet/datagram/mod.rs index 7b638de5e..b9b577e5e 100644 --- a/kernel/src/net/socket/inet/datagram/mod.rs +++ b/kernel/src/net/socket/inet/datagram/mod.rs @@ -202,7 +202,7 @@ pub struct UdpSocketData { impl UdpSocket { pub fn new(nonblock: bool, version: IpVersion) -> Result, SystemError> { let netns = ProcessManager::current_netns(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let task = ProcessManager::current_pcb(); let account = crate::cgroup::memory::account_for_task(&task); @@ -445,7 +445,7 @@ impl UdpSocket { &self, iface: Arc, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match super::common::registration::PreparedIfaceSocket::try_new( @@ -747,7 +747,7 @@ impl UdpSocket { if limit > 1 << 30 { return Err(SystemError::ENOBUFS); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let operation = self.resize_lock.lock(); // IPv4 RX belongs to receive_queue, not the TX-only smoltcp entry. diff --git a/kernel/src/net/socket/inet/raw/socket.rs b/kernel/src/net/socket/inet/raw/socket.rs index 91cccb569..0a2cdb970 100644 --- a/kernel/src/net/socket/inet/raw/socket.rs +++ b/kernel/src/net/socket/inet/raw/socket.rs @@ -69,7 +69,7 @@ impl RawSocket { // 但为了让未 bind 的 raw socket 能接收数据包、且 poll/epoll 能正确唤醒， // 在存在 iface 时优先以“通配接收”方式附着到 loopback/默认 iface。 // 若当前 netns 尚无可用 iface，则退化为 Unbound，允许后续 bind/connect/sendto 再完成选址与附着。 - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let (memory, body, unbound, sock, me, send_account) = loop { let task = ProcessManager::current_pcb(); @@ -193,7 +193,7 @@ impl RawSocket { &self, iface: Arc, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match super::super::common::registration::PreparedIfaceSocket::try_new( diff --git a/kernel/src/net/socket/inet/stream/lifecycle.rs b/kernel/src/net/socket/inet/stream/lifecycle.rs index 71eb15f05..182204112 100644 --- a/kernel/src/net/socket/inet/stream/lifecycle.rs +++ b/kernel/src/net/socket/inet/stream/lifecycle.rs @@ -346,7 +346,7 @@ impl TcpSocket { } pub fn do_listen(&self, backlog: usize) -> Result<(), SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let unbound = matches!( self.inner.read().as_ref(), @@ -476,7 +476,7 @@ impl TcpSocket { pub fn try_accept(&self) -> Result<(Arc, smoltcp::wire::IpEndpoint), SystemError> { // Poll outside `inner`: protocol notification may acquire it again. self.netns().tcp_stack().poll(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let (handle, identity) = { let reader = self.inner.read(); diff --git a/kernel/src/net/socket/inet/stream/stream_core.rs b/kernel/src/net/socket/inet/stream/stream_core.rs index f73c5db57..1b4ab4002 100644 --- a/kernel/src/net/socket/inet/stream/stream_core.rs +++ b/kernel/src/net/socket/inet/stream/stream_core.rs @@ -255,7 +255,7 @@ impl TcpSocket { pub(super) fn prepare_bound_slot( &self, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match self.netns.tcp_stack().reserve_bound_slot(&creator) { @@ -361,7 +361,7 @@ impl TcpSocket { pub fn new(nonblock: bool, ver: smoltcp::wire::IpVersion) -> Result, SystemError> { let netns = ProcessManager::current_netns(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let task = ProcessManager::current_pcb(); let account = crate::cgroup::memory::account_for_task(&task); @@ -471,7 +471,7 @@ impl TcpSocket { Some(inner::Inner::Init(inner::Init::Unbound(_))) ) }; - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); while needs_handle() { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match self.netns.tcp_stack().prepare_socket_capacity(&creator) { @@ -710,7 +710,7 @@ impl TcpSocket { } fn update_socket_buffer(&self, size: usize, send: bool) -> Result<(), SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { // Domain lookup precedes even the task-only resize mutex. let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); diff --git a/kernel/src/net/socket/unix/datagram/mod.rs b/kernel/src/net/socket/unix/datagram/mod.rs index 8e6af4a62..fe1d78002 100644 --- a/kernel/src/net/socket/unix/datagram/mod.rs +++ b/kernel/src/net/socket/unix/datagram/mod.rs @@ -688,7 +688,7 @@ impl Socket for UnixDatagramSocket { fn bind(&self, endpoint: Endpoint) -> Result<(), SystemError> { let unix_endpoint = UnixEndpoint::try_from(endpoint)?; - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let (mut inner, was_unbound) = loop { let candidate = unix_endpoint.clone(); let mut inner = self.inner.lock(); diff --git a/kernel/src/net/socket/unix/mod.rs b/kernel/src/net/socket/unix/mod.rs index 5ef0873ce..b99845b9a 100644 --- a/kernel/src/net/socket/unix/mod.rs +++ b/kernel/src/net/socket/unix/mod.rs @@ -456,29 +456,13 @@ impl UnixEndpoint { path.trim(), )?; // All connect_in callers resolve before taking socket/backend - // guards. Lookup has not published the connection, so quota + // guards. Lookup has not published the connection, so allocation // recovery may retry this read-only resolution stage only. - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let inode = loop { match inode_begin.lookup_follow_symlink(&path, VFS_MAX_FOLLOW_SYMLINK_TIMES) { Ok(inode) => break inode, - Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { - return Err(error); - } - Err(crate::mm::allocation::MemoryAllocationError::Charge(failure)) => { - use crate::cgroup::memory::runtime::{ - recover_charge, MemoryAllocPolicy, RecoveryDecision, - }; - match recover_charge( - &failure, - MemoryAllocPolicy::ReclaimThenOom, - 0, - &mut recovery, - )? { - RecoveryDecision::RetryOperation => continue, - RecoveryDecision::Exhausted => return Err(SystemError::ENOMEM), - } - } + Err(error) => error.recover(&mut recovery)?, } }; if inode.metadata()?.file_type != FileType::Socket { diff --git a/kernel/src/net/socket/unix/stream/mod.rs b/kernel/src/net/socket/unix/stream/mod.rs index 9a769013a..0d1350111 100644 --- a/kernel/src/net/socket/unix/stream/mod.rs +++ b/kernel/src/net/socket/unix/stream/mod.rs @@ -747,7 +747,7 @@ impl Socket for UnixStreamSocket { fn bind(&self, endpoint: Endpoint) -> Result<(), SystemError> { let addr = UnixEndpoint::try_from(endpoint)?; - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let candidate = addr.clone(); let result = { diff --git a/kernel/src/net/syscall/mod.rs b/kernel/src/net/syscall/mod.rs index 246075837..d0d0f4df9 100644 --- a/kernel/src/net/syscall/mod.rs +++ b/kernel/src/net/syscall/mod.rs @@ -26,7 +26,7 @@ pub(super) fn reserve_socket_descriptors( task: &alloc::sync::Arc, cloexec: bool, ) -> Result, system_error::SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(task); match task @@ -43,7 +43,7 @@ pub(super) fn reserve_socket_descriptors( /// socket pair. Descriptor reservations are owners, not held fd-table locks. pub(super) fn prepare_socket_file( ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { match crate::filesystem::vfs::file::PreparedFile::try_new() { Ok(prepared) => return Ok(prepared), @@ -61,7 +61,7 @@ pub(super) fn finish_socket_file( flags: crate::filesystem::vfs::file::FileFlags, mut prepared: crate::filesystem::vfs::file::PreparedFile, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let result = crate::filesystem::vfs::file::File::new_socket_prepared(inode.clone(), flags, prepared); diff --git a/kernel/src/net/syscall/sys_socketpair.rs b/kernel/src/net/syscall/sys_socketpair.rs index cfbbaabd6..2823b4f5a 100644 --- a/kernel/src/net/syscall/sys_socketpair.rs +++ b/kernel/src/net/syscall/sys_socketpair.rs @@ -154,7 +154,7 @@ pub(super) fn do_socketpair( ) { return Err(SystemError::ESOCKTNOSUPPORT); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let (prepared_a, prepared_b) = loop { let a = match crate::filesystem::vfs::file::PreparedFile::try_new() { Ok(storage) => storage, diff --git a/kernel/src/process/exec.rs b/kernel/src/process/exec.rs index 2900bc2d3..d2312025c 100644 --- a/kernel/src/process/exec.rs +++ b/kernel/src/process/exec.rs @@ -399,7 +399,7 @@ impl ExecParam { if me.sighand().is_shared() { let old_sighand = me.sighand(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let new_sighand = loop { let creator = crate::cgroup::memory::object_for_task(&me); match SigHand::try_new_for(&creator) { @@ -418,7 +418,7 @@ impl ExecParam { .fd_table_snapshot() .expect("exec task has no fd table"); if shared { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let private = loop { let creator = crate::cgroup::memory::object_for_task(&me); match FileDescriptorTable::try_clone(&fd_table, None, &creator) { diff --git a/kernel/src/process/fork.rs b/kernel/src/process/fork.rs index c0ef768e6..b9990b9fb 100644 --- a/kernel/src/process/fork.rs +++ b/kernel/src/process/fork.rs @@ -385,7 +385,7 @@ impl ProcessManager { .flags .contains(CloneFlags::CLONE_PIDFD) .then(crate::mm::ucontext::AddressSpace::current); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { // Each failed unpublished birth releases both stacks and its // body before recovery. Never enter copy_process locks to reclaim. diff --git a/kernel/src/process/kstack.rs b/kernel/src/process/kstack.rs index 3255fefcb..999f738af 100644 --- a/kernel/src/process/kstack.rs +++ b/kernel/src/process/kstack.rs @@ -190,6 +190,7 @@ impl KernelStack { pub fn new() -> Result { Self::new_with_account(None).map_err(|error| match error { crate::mm::allocation::MemoryAllocationError::System(error) => error, + crate::mm::allocation::MemoryAllocationError::Physical => SystemError::ENOMEM, crate::mm::allocation::MemoryAllocationError::Charge(_) => { unreachable!("unaccounted stack cannot fail admission") } @@ -216,7 +217,14 @@ impl KernelStack { .transpose()?; if cfg!(feature = "kstack_protect") { unsafe { - let (kstack_vaddr, kstack_paddr, count) = alloc_from_kernel_space()?; + let (kstack_vaddr, kstack_paddr, count) = + alloc_from_kernel_space().map_err(|error| { + if error == SystemError::ENOMEM { + crate::mm::allocation::MemoryAllocationError::Physical + } else { + error.into() + } + })?; let real_kstack_vaddr = kstack_vaddr + KernelStack::SIZE; Ok(Self { stack: Some( @@ -234,7 +242,7 @@ impl KernelStack { stack: None, dynamic_stack: Some(unsafe { PhysicalBox::::try_new_zeroed() - .map_err(|_| SystemError::ENOMEM)? + .map_err(|_| crate::mm::allocation::MemoryAllocationError::Physical)? .assume_init() }), ty: KernelStackType::Dynamic, diff --git a/kernel/src/process/namespace/setns.rs b/kernel/src/process/namespace/setns.rs index ee03b699f..0d3d0be67 100644 --- a/kernel/src/process/namespace/setns.rs +++ b/kernel/src/process/namespace/setns.rs @@ -68,7 +68,7 @@ fn install_setns( new_cred: Option>, flags: CloneFlags, ) -> Result<(), SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let mut new_fs = None; let fs_refs = lock_fs_refs_copy(); diff --git a/kernel/src/process/namespace/unshare.rs b/kernel/src/process/namespace/unshare.rs index 1616a938a..a92c8d22c 100644 --- a/kernel/src/process/namespace/unshare.rs +++ b/kernel/src/process/namespace/unshare.rs @@ -21,7 +21,7 @@ pub fn ksys_unshare(flags: CloneFlags) -> Result<(), SystemError> { check_unshare_flags(flags)?; let current_pcb = ProcessManager::current_pcb(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let new_cred = unshare_user_cred(flags, ¤t_pcb)?.map(Cred::new_arc); // Retain completed candidates outside the topology barrier, even if diff --git a/kernel/src/process/pid_selftest.rs b/kernel/src/process/pid_selftest.rs index cb8bf27c6..4ea4fab4b 100644 --- a/kernel/src/process/pid_selftest.rs +++ b/kernel/src/process/pid_selftest.rs @@ -5,7 +5,7 @@ use super::*; use crate::process::KernelStack; pub(crate) fn task() -> Result, SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let task = loop { let birth = (|| { let resources = crate::process::task::PreparedTaskResources::try_for_fork( diff --git a/kernel/src/process/posix_timer/cpu.rs b/kernel/src/process/posix_timer/cpu.rs index 5e76e1a4a..c8699512b 100644 --- a/kernel/src/process/posix_timer/cpu.rs +++ b/kernel/src/process/posix_timer/cpu.rs @@ -28,7 +28,7 @@ impl CpuBackend { } else { task.process_signal().cpu_event_domain() }; - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); let registration = loop { match CpuEventDomain::prepare_registration(&object_for_task(pcb), callback.clone()) { Ok(registration) => break registration, diff --git a/kernel/src/process/syscall/sys_pidfdopen.rs b/kernel/src/process/syscall/sys_pidfdopen.rs index cb3d0621d..be355d4db 100644 --- a/kernel/src/process/syscall/sys_pidfdopen.rs +++ b/kernel/src/process/syscall/sys_pidfdopen.rs @@ -54,7 +54,7 @@ impl Syscall for SysPidFdOpen { if pid.pid_task(PidType::TGID).is_none() { return Err(SystemError::EINVAL); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let prepared = loop { match PidFd::prepare(¤t, pid.clone(), file_flags, false) { Ok(prepared) => break prepared, diff --git a/kernel/src/process/usermodehelper.rs b/kernel/src/process/usermodehelper.rs index ddb5282d4..80cb443bc 100644 --- a/kernel/src/process/usermodehelper.rs +++ b/kernel/src/process/usermodehelper.rs @@ -214,7 +214,7 @@ fn run_child_prepare( if let Some(cred) = cred { child.install_cred(cred); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let fd_table = loop { // No PCB basic-info, fd-table, or helper-result guard spans admission // or recovery. Preserve the old table until the private one is ready. diff --git a/kernel/src/syscall/user_access.rs b/kernel/src/syscall/user_access.rs index 729c48f72..b4020afba 100644 --- a/kernel/src/syscall/user_access.rs +++ b/kernel/src/syscall/user_access.rs @@ -102,7 +102,7 @@ fn prefault_user_range_mode( }; let mut current = start_page; let mut retried = false; - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); 'retry: loop { let creator = crate::cgroup::memory::object_for_task( &crate::process::ProcessManager::current_pcb(), @@ -186,6 +186,19 @@ fn prefault_user_range_mode( )?; continue 'retry; } + Err( + error @ crate::mm::allocation::MemoryAllocationError::Physical, + ) => { + if let Err(error) = error.recover(&mut charge_recovery) { + crate::mm::fault::account_fault_recovery_failure( + &data_account, + fault_flags, + error.clone(), + ); + return Err(error); + } + continue 'retry; + } Err(crate::mm::allocation::MemoryAllocationError::System( error, )) => Err(error), diff --git a/kernel/src/time/cpu_events/mod.rs b/kernel/src/time/cpu_events/mod.rs index a8f20862c..a3b6efa9e 100644 --- a/kernel/src/time/cpu_events/mod.rs +++ b/kernel/src/time/cpu_events/mod.rs @@ -3,6 +3,7 @@ //! Producers only publish counters and schedule one preborn tasklet. The lock //! order is domain queues -> registration state. Callbacks and final drops are //! outside both guards; registrations do not retain their domain. +use crate::mm::allocation::AllocationRecoveryState; mod queue; use alloc::sync::Arc; @@ -10,10 +11,7 @@ use core::sync::atomic::{fence, AtomicU64, Ordering}; use system_error::SystemError; use crate::{ - cgroup::memory::{ - accounting::KernelByteKind, object::ObjectAccount, owned::OwnedBox, - runtime::ChargeRecoveryState, - }, + cgroup::memory::{accounting::KernelByteKind, object::ObjectAccount, owned::OwnedBox}, exception::tasklet::{tasklet_schedule, Tasklet}, libs::{ spinlock::SpinLock, @@ -349,7 +347,7 @@ impl CpuEventDomain { return Ok(()); } let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); - let mut recovery = ChargeRecoveryState::new(); + let mut recovery = AllocationRecoveryState::new(); loop { match self.wait_once(kind, deadline, &mut condition, &creator) { Ok(()) => return Ok(()), diff --git a/user/apps/tests/dunitest/suites/normal/global_reclaim.cc b/user/apps/tests/dunitest/suites/normal/global_reclaim.cc index 046920b98..fbb01fb2c 100644 --- a/user/apps/tests/dunitest/suites/normal/global_reclaim.cc +++ b/user/apps/tests/dunitest/suites/normal/global_reclaim.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -9,9 +10,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -30,10 +33,93 @@ uint64_t MemTotalKiB() { // Keep this capacity workload unchanged between baseline and fixed kernels. // Dirty mode additionally exercises writeback progress before physical retry. -int ExerciseReclaim(int fd, bool dirty) { - constexpr size_t file_bytes = 192 * kMiB; +int ExerciseReclaim(int fd, bool dirty, bool syscall_io) { + const size_t file_bytes = (syscall_io ? 128 : 192) * kMiB; constexpr size_t anonymous_bytes = 700 * kMiB; if (ftruncate(fd, file_bytes) != 0) return 10; + if (syscall_io) { + // Materialize extents before pressure so this regression measures cache + // admission rather than per-block sparse-growth metadata transactions. + // The pressure phase must still allocate cache pages for these extents. + { + unsigned char initial[64 * 1024] = {}; + for (size_t offset = 0; offset < file_bytes; offset += sizeof(initial)) { + if (pwrite(fd, initial, sizeof(initial), offset) != sizeof(initial)) + return 29; + } + if (fsync(fd) != 0) return 30; + if (posix_fadvise(fd, 0, file_bytes, POSIX_FADV_DONTNEED) != 0) return 31; + } + fprintf(stderr, "file reclaim preparation complete\n"); + // Keep anonymous memory resident while the backing cache competes for + // frames. Unlike the mmap tests below, every new file page is allocated + // by pread/pwrite, outside the user page-fault recovery path. + void* anonymous = mmap(nullptr, anonymous_bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anonymous == MAP_FAILED) return 20; + auto* resident = static_cast(anonymous); + for (size_t offset = 0; offset < anonymous_bytes; offset += 4096) + resident[offset] = 37; + unsigned char buffer[4096]; + for (size_t offset = 0; offset < file_bytes; offset += sizeof(buffer)) { + const unsigned char pattern = + static_cast(offset / sizeof(buffer)); + memset(buffer, pattern, sizeof(buffer)); + const ssize_t written = pwrite(fd, buffer, sizeof(buffer), offset); + if (written != sizeof(buffer)) { + const int error = written < 0 ? errno : 0; + fprintf(stderr, "pwrite offset=%zu result=%zd errno=%d\n", + offset, written, error); + return 21; + } + } + fprintf(stderr, "file reclaim write complete\n"); + if (fsync(fd) != 0) { + fprintf(stderr, "fsync errno=%d\n", errno); + return 22; + } + fprintf(stderr, "file reclaim fsync complete\n"); + // Exercise short thread lifetimes while anonymous memory remains + // resident and the file cache has just been populated. This covers + // usability under pressure, not deterministic stack fragmentation. + for (unsigned int attempt = 0; attempt < 4; ++attempt) { + pthread_t thread; + int error = pthread_create( + &thread, nullptr, [](void* value) -> void* { return value; }, buffer); + if (error != 0) { + fprintf(stderr, "pthread_create attempt=%u error=%d\n", attempt, error); + return 27; + } + void* returned = nullptr; + error = pthread_join(thread, &returned); + if (error != 0 || returned != buffer) { + fprintf(stderr, "pthread_join attempt=%u error=%d value_matches=%d\n", + attempt, error, returned == buffer); + return 28; + } + } + fprintf(stderr, "file reclaim threads complete\n"); + for (size_t end = file_bytes; end != 0; end -= sizeof(buffer)) { + const size_t offset = end - sizeof(buffer); + const unsigned char pattern = + static_cast(offset / sizeof(buffer)); + const ssize_t read = pread(fd, buffer, sizeof(buffer), offset); + if (read != sizeof(buffer)) { + const int error = read < 0 ? errno : 0; + fprintf(stderr, "pread offset=%zu result=%zd errno=%d\n", + offset, read, error); + return 23; + } + for (unsigned char value : buffer) { + if (value != pattern) return 24; + } + } + fprintf(stderr, "file reclaim read complete\n"); + for (size_t offset = 0; offset < anonymous_bytes; offset += 4096) { + if (resident[offset] != 37) return 25; + } + return munmap(anonymous, anonymous_bytes) == 0 ? 0 : 26; + } void* file = mmap(nullptr, file_bytes, PROT_READ | (dirty ? PROT_WRITE : 0), MAP_SHARED, fd, 0); if (file == MAP_FAILED) return 11; @@ -83,7 +169,7 @@ bool WriteText(const std::string& path, const std::string& value) { return written && closed; } -void CheckReclaim(bool dirty, bool protect_low = false) { +void CheckReclaim(bool dirty, bool protect_low = false, bool syscall_io = false) { // A capacity regression, run in the dedicated 1 GiB guest. Larger guests // would pass without entering physical reclaim and provide false coverage. const uint64_t total = MemTotalKiB(); @@ -135,7 +221,7 @@ void CheckReclaim(bool dirty, bool protect_low = false) { if (child == 0) { if (protect_low && !WriteText(group + "/cgroup.procs", std::to_string(getpid()))) _exit(19); - _exit(ExerciseReclaim(fd, dirty)); + _exit(ExerciseReclaim(fd, dirty, syscall_io)); } if (child < 0) { close(fd); @@ -146,7 +232,12 @@ void CheckReclaim(bool dirty, bool protect_low = false) { } int status = 0; pid_t result = 0; - for (int i = 0; i < 12000 && result == 0; ++i) { + // The syscall case includes extent preparation and two full syncs. This is + // its whole-test budget, independent of the kernel's physical retry bound. + const int timeout_seconds = syscall_io ? 300 : 120; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(timeout_seconds); + while (result == 0 && std::chrono::steady_clock::now() < deadline) { result = waitpid(child, &status, WNOHANG); if (result < 0 && errno == EINTR) result = 0; if (result == 0) usleep(10000); @@ -167,10 +258,12 @@ void CheckReclaim(bool dirty, bool protect_low = false) { EXPECT_GT(low, 0u) << "must exercise low-protection fallback"; events.close(); EXPECT_EQ(rmdir(group.c_str()), 0) << errno; - if (enabled_memory) + if (enabled_memory) { EXPECT_TRUE(WriteText("/sys/fs/cgroup/cgroup.subtree_control", "-memory")); + } } - ASSERT_EQ(result, child) << "reclaim child failed to finish within 120s"; + ASSERT_EQ(result, child) << "reclaim child failed to finish within " + << timeout_seconds << "s"; ASSERT_TRUE(WIFEXITED(status)) << "unexpected signal: " << status; EXPECT_EQ(WEXITSTATUS(status), 0); EXPECT_EQ(OomKills(), before) << "reclaimable cache must not cause an OOM kill"; @@ -183,6 +276,12 @@ TEST(GlobalReclaim, CleanCacheRemainsReclaimableWhenPreparationCannotAllocate) { TEST(GlobalReclaim, DirtyCacheMakesProgressBeforeGlobalOom) { CheckReclaim(true); } +TEST(GlobalReclaim, FileSyscallAllocationsRecoverUnderGlobalPressure) { + // 128 MiB crosses the reproduced file-allocation failure point while + // keeping this syscall regression bounded. The separate 192 MiB mmap + // cases above exercise the larger reclaim inventory. + CheckReclaim(false, false, true); +} TEST(GlobalReclaim, GlobalPressureCanBreachLowAcrossBoundedScanBatches) { // 192 MiB is 49,152 pages, well beyond one 4,096-candidate scan. // memory.low must yield under global pressure; memory.min remains hard. From e11dd1c13b511453328080220006727e7b525a4e Mon Sep 17 00:00:00 2001 From: longjin Date: Wed, 4 Nov 2026 02:35:40 +0000 Subject: [PATCH 2/2] fix(mm): decouple allocation recovery eligibility from vmalloc Move the unchanged task/IRQ/preemption eligibility predicate to the always available MM allocation module. Re-export it for existing vmalloc callers so LoongArch recovery no longer depends on an architecture-gated VA backend. Preserve the initialization short-circuit and the existing NOIO policy. Correct the socket and ELF scratch birth fixtures to check a real single Physical allocation refusal before delivering its terminal errno to the consumer. Keep one-attempt, no-consumption, file-position and accounting assertions; do not require the production recovery loop to make only one attempt against an intentionally permanent refusal. Reproduced the original LoongArch compile error and both failing retirement checks before the fix. Validated LoongArch check and release build, default and CI-feature x86 kernel builds, formatting/all-features clippy, and 45 guest cases including the complete retirement report and related MM/VFS/loop suites. Signed-off-by: longjin --- kernel/src/libs/elf/scratch_selftest.rs | 27 ++++++++++--------- kernel/src/mm/allocation.rs | 17 ++++++++++-- kernel/src/mm/vmalloc.rs | 10 ++----- .../src/net/socket/base/scratch_selftest.rs | 23 ++++++++++------ 4 files changed, 47 insertions(+), 30 deletions(-) diff --git a/kernel/src/libs/elf/scratch_selftest.rs b/kernel/src/libs/elf/scratch_selftest.rs index d5596feb3..0a8baec4f 100644 --- a/kernel/src/libs/elf/scratch_selftest.rs +++ b/kernel/src/libs/elf/scratch_selftest.rs @@ -9,7 +9,7 @@ use crate::{ FileSystem, FileType, InodeMode, }, }, - mm::kernel_buffer::KernelBuffer, + mm::{allocation::MemoryAllocationError, kernel_buffer::KernelBuffer}, }; pub(crate) fn run() -> bool { @@ -31,18 +31,21 @@ pub(crate) fn run() -> bool { ElfLoader::FILE_READ_BUF_SIZE + 1, 0, &file, - |size| { + |size| -> Result { requested = size; - File::recover_io(|| { - KernelBuffer::try_new_zeroed_in_with( - size, - prepared.object_account(), - &mut || { - backing_attempts += 1; - Err(SystemError::ENOMEM) - }, - ) - }) + // This fixture checks a single refused scratch birth before + // population, independently of global recovery policy. + match KernelBuffer::try_new_zeroed_in_with( + size, + prepared.object_account(), + &mut || { + backing_attempts += 1; + Err(SystemError::ENOMEM) + }, + ) { + Err(error @ MemoryAllocationError::Physical) => Err(error.into_errno()), + _ => Err(SystemError::EINVAL), + } }, ); Ok(matches!(result, Err(SystemError::ENOMEM)) diff --git a/kernel/src/mm/allocation.rs b/kernel/src/mm/allocation.rs index c1f7fe82e..0c33fa393 100644 --- a/kernel/src/mm/allocation.rs +++ b/kernel/src/mm/allocation.rs @@ -7,6 +7,19 @@ use crate::cgroup::memory::{ runtime::{recover_charge, ChargeRecoveryState, MemoryAllocPolicy, RecoveryDecision}, }; use crate::mm::oom_reaper::JobBirthError; +use crate::{ + arch::CurrentIrqArch, + exception::{in_interrupt, InterruptArch}, + process::ProcessManager, +}; + +/// Common eligibility for sleepable allocation work, independent of the VA backend. +pub(crate) fn task_context() -> bool { + CurrentIrqArch::is_irq_enabled() + && !in_interrupt() + && ProcessManager::initialized() + && ProcessManager::current_pcb().preempt_count() == 0 +} #[derive(Debug)] pub(crate) enum MemoryAllocationError { @@ -56,7 +69,7 @@ impl AllocationRecoveryState { fn recover_physical(&mut self) -> Result<(), SystemError> { // Check task/IRQ/preemption before reading any task-local NOIO state. - if !super::vmalloc::task_context() || crate::cgroup::memory::io::noio_active() { + if !task_context() || crate::cgroup::memory::io::noio_active() { return Err(SystemError::ENOMEM); } let deadline = *self @@ -123,7 +136,7 @@ impl MemoryAllocationError { #[cfg(feature = "mm_selftest")] pub(crate) fn recovery_selftest() -> bool { - if !super::vmalloc::task_context() { + if !task_context() { return false; } let mut state = AllocationRecoveryState::new(); diff --git a/kernel/src/mm/vmalloc.rs b/kernel/src/mm/vmalloc.rs index 1f31992ec..947eb9a42 100644 --- a/kernel/src/mm/vmalloc.rs +++ b/kernel/src/mm/vmalloc.rs @@ -4,6 +4,7 @@ //! uses only non-recursive local metadata storage; interrupt-side destruction //! transfers its existing record to the permanent memory notification worker. +pub(crate) use super::allocation::task_context; #[cfg(feature = "mm_selftest")] use crate::cgroup::memory::object_for_task; use crate::{ @@ -12,7 +13,7 @@ use crate::{ accounting::KernelByteKind, object::{ObjectAccount, ObjectCharge}, }, - exception::{in_interrupt, InterruptArch}, + exception::InterruptArch, libs::{ rbtree::{PreparedNode, RBTree}, spinlock::SpinLock, @@ -294,13 +295,6 @@ impl DerefMut for VmAllocation { } } -pub(crate) fn task_context() -> bool { - CurrentIrqArch::is_irq_enabled() - && !in_interrupt() - && ProcessManager::initialized() - && ProcessManager::current_pcb().preempt_count() == 0 -} - #[derive(Clone, Copy)] struct Plan { pages: usize, diff --git a/kernel/src/net/socket/base/scratch_selftest.rs b/kernel/src/net/socket/base/scratch_selftest.rs index 13dc9f7c9..f40aad79f 100644 --- a/kernel/src/net/socket/base/scratch_selftest.rs +++ b/kernel/src/net/socket/base/scratch_selftest.rs @@ -2,8 +2,7 @@ use super::*; use crate::{ cgroup::memory::{object::ObjectAccount, MemoryAccount}, - filesystem::vfs::file::File, - mm::kernel_buffer::KernelBuffer, + mm::{allocation::MemoryAllocationError, kernel_buffer::KernelBuffer}, net::socket::unix::datagram::UnixDatagramSocket, syscall::user_buffer::UserBuffer, }; @@ -19,14 +18,22 @@ pub(crate) fn run() -> bool { // No copy may occur: the real backing allocation below fails first. let mut output = UserBuffer::new_protected(0x1000 as *mut u8, 16384, true)?; let mut backing_attempts = 0; - let result = read_to_user_buffer_with(receiver.as_ref(), &mut output, 16384, |size| { - File::recover_io(|| { - KernelBuffer::try_new_zeroed_in_with(size, owner, &mut || { + let result = read_to_user_buffer_with( + receiver.as_ref(), + &mut output, + 16384, + |size| -> Result { + // Check one refused birth and consumer state here; bounded + // global recovery policy has separate regression coverage. + match KernelBuffer::try_new_zeroed_in_with(size, owner, &mut || { backing_attempts += 1; Err(SystemError::ENOMEM) - }) - }) - }); + }) { + Err(error @ MemoryAllocationError::Physical) => Err(error.into_errno()), + _ => Err(SystemError::EINVAL), + } + }, + ); if !matches!(result, Err(SystemError::ENOMEM)) || backing_attempts != 1 { return Ok(false); }