diff --git a/kernel/src/arch/riscv64/mm/fault.rs b/kernel/src/arch/riscv64/mm/fault.rs index 094f7a8de..a13a1abde 100644 --- a/kernel/src/arch/riscv64/mm/fault.rs +++ b/kernel/src/arch/riscv64/mm/fault.rs @@ -1,6 +1,6 @@ +use crate::mm::allocation::AllocationRecoveryState; use crate::{ arch::ipc::signal::Signal, - cgroup::memory::runtime::ChargeRecoveryState, ipc::signal_types::{BUS_ADRERR, SEGV_ACCERR, SEGV_MAPERR}, mm::{ allocation::MemoryAllocationError, @@ -72,7 +72,7 @@ impl RiscV64MMArch { } _ => unreachable!("not a user page fault"), }; - let mut charge_recovery = ChargeRecoveryState::new(); + let mut charge_recovery = AllocationRecoveryState::new(); let mut global_recovery = GlobalFaultRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); @@ -122,7 +122,8 @@ impl RiscV64MMArch { Err(MemoryAllocationError::System( SystemError::EINTR | SystemError::ERESTARTSYS, )) => return Ok(()), - Err(MemoryAllocationError::System(SystemError::ENOMEM)) => { + Err(MemoryAllocationError::System(SystemError::ENOMEM)) + | Err(MemoryAllocationError::Physical) => { reason = VmFaultReason::VM_FAULT_OOM } Err(MemoryAllocationError::System(_)) => { @@ -179,7 +180,7 @@ impl RiscV64MMArch { /// recovery yields to pending signals and a later hardware fault, as on x86. fn recover_charge( failure: crate::cgroup::memory::accounting::ChargeFailure, - state: &mut ChargeRecoveryState, + state: &mut AllocationRecoveryState, account: &crate::cgroup::memory::MemoryAccount, flags: FaultFlags, ) -> bool { diff --git a/kernel/src/arch/x86_64/mm/fault.rs b/kernel/src/arch/x86_64/mm/fault.rs index 503f94072..7f1e2f80d 100644 --- a/kernel/src/arch/x86_64/mm/fault.rs +++ b/kernel/src/arch/x86_64/mm/fault.rs @@ -419,7 +419,7 @@ impl X86_64MMArch { }; let mut global_recovery = crate::mm::fault_recovery::GlobalFaultRecoveryState::new(); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); 'fault_retry: loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); @@ -717,7 +717,8 @@ impl X86_64MMArch { } Err(crate::mm::allocation::MemoryAllocationError::System( SystemError::ENOMEM, - )) => { + )) + | Err(crate::mm::allocation::MemoryAllocationError::Physical) => { fault = VmFaultReason::VM_FAULT_OOM; crate::mm::fault::account_fault_result(&data_account, flags, fault); space_guard = current_address_space.write(); diff --git a/kernel/src/driver/base/block/gendisk/cache.rs b/kernel/src/driver/base/block/gendisk/cache.rs index acc21d481..28781c6fa 100644 --- a/kernel/src/driver/base/block/gendisk/cache.rs +++ b/kernel/src/driver/base/block/gendisk/cache.rs @@ -52,10 +52,9 @@ where }; let confirmed_mask = (((1u16 << confirmed_last) - 1) ^ ((1u16 << first) - 1)) as u8; *remaining &= !confirmed_mask; - result.sync_result.map_err(|error| match error { - MemoryAllocationError::System(error) => error, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + result + .sync_result + .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?; if result.written_len != end - start { return Err(SystemError::EIO); } @@ -149,10 +148,9 @@ impl PageCacheBackend for RawBlockBackend { // A completed result belongs to the lower owner's exact-sync protocol // (loop's write_piece already performs that recovery); never replay it. let result = File::recover_io(|| disk.write_payload(offset, bytes, WriteSyncIntent::None))?; - result.sync_result.map_err(|error| match error { - MemoryAllocationError::System(error) => error, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + result + .sync_result + .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?; Ok(result.written_len) } diff --git a/kernel/src/driver/base/block/gendisk/mod.rs b/kernel/src/driver/base/block/gendisk/mod.rs index a89597421..ac2ca5d01 100644 --- a/kernel/src/driver/base/block/gendisk/mod.rs +++ b/kernel/src/driver/base/block/gendisk/mod.rs @@ -280,10 +280,9 @@ impl GenDisk { })?; // The lower owner has already submitted this prefix and owns exact // sync recovery. Its completion error must never replay the payload. - result.sync_result.map_err(|error| match error { - crate::mm::allocation::MemoryAllocationError::System(error) => error, - crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + result + .sync_result + .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?; Ok(result.written_len) } diff --git a/kernel/src/driver/block/loop_device/io.rs b/kernel/src/driver/block/loop_device/io.rs index 74b49cde1..6ef460f61 100644 --- a/kernel/src/driver/block/loop_device/io.rs +++ b/kernel/src/driver/block/loop_device/io.rs @@ -168,8 +168,19 @@ impl LoopDevice { }) .map_err(Self::backing_error)?; result.sync_result = Self::normalize_backing_sync_result(result.sync_result); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); while let Err(error) = core::mem::replace(&mut result.sync_result, Ok(())) { + if matches!(error, MemoryAllocationError::Physical) { + // The loop pin/gate and NOIO scope still belong to our caller. + // Let its lock-free boundary finish the retained sync range; + // never consume the deferred completions or replay payload. + result.sync_result = Err(if result.sync_range.is_some() { + error + } else { + error.into_errno().into() + }); + break; + } // Payload is committed: recover only its retained exact sync range. if let Err(error) = error.recover(&mut recovery) { result.sync_result = Err(error.into()); @@ -254,7 +265,12 @@ impl LoopDevice { Ok(result) => result, // This is a data error, not a recoverable sync failure. No // invented sync_range may replay an unsubmitted remainder. - Err(error) if done != 0 => return Ok(DelegatedWriteResult::new(done, Err(error))), + Err(error) if done != 0 => { + return Ok(DelegatedWriteResult::new( + done, + Err(error.into_errno().into()), + )) + } Err(error) => return Err(error), }; let expected = if full { count } else { LBA_SIZE }; @@ -317,15 +333,35 @@ impl LoopDevice { fn recover_backing_io( mut attempt: impl FnMut() -> Result, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); loop { match attempt() { Ok(value) => return Ok(value), + // Every caller retains loop admission and a NOIO scope. A + // physical failure must escape with its provenance intact so + // the outer transaction can reclaim after releasing them. + Err(error @ MemoryAllocationError::Physical) => return Err(error), Err(error) => error.recover(&mut recovery)?, } } } + #[cfg(feature = "mm_selftest")] + pub(crate) fn physical_boundary_selftest() -> bool { + let _noio = crate::cgroup::memory::io::MemoryIoScope::enter_noio(); + let mut attempts = 0; + let result: Result<(), MemoryAllocationError> = Self::recover_backing_io(|| { + attempts += 1; + Err(MemoryAllocationError::Physical) + }); + attempts == 1 + && matches!(result, Err(MemoryAllocationError::Physical)) + && matches!( + Self::normalize_backing_sync_result(Err(MemoryAllocationError::Physical)), + Err(MemoryAllocationError::Physical) + ) + } + fn normalize_backing_sync_result( result: Result<(), MemoryAllocationError>, ) -> Result<(), MemoryAllocationError> { @@ -333,7 +369,9 @@ impl LoopDevice { // it to BLK_STS_IOERR. No backing flush error is a successful flush. match result { Ok(()) => Ok(()), - Err(error @ MemoryAllocationError::Charge(_)) => Err(error), + Err(error @ (MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical)) => { + Err(error) + } Err(MemoryAllocationError::System(_)) => Err(SystemError::EIO.into()), } } @@ -361,14 +399,16 @@ impl LoopDevice { | SystemError::ETIME => error.into(), _ => SystemError::EIO.into(), }, - error @ MemoryAllocationError::Charge(_) => error, + error @ (MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => error, } } pub(super) fn legacy_io_terminal(error: MemoryAllocationError) -> SystemError { match error { MemoryAllocationError::System(error) => error, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, + MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical => { + SystemError::ENOMEM + } } } } diff --git a/kernel/src/filesystem/epoll/event_poll.rs b/kernel/src/filesystem/epoll/event_poll.rs index 4ca03428c..17c11184a 100644 --- a/kernel/src/filesystem/epoll/event_poll.rs +++ b/kernel/src/filesystem/epoll/event_poll.rs @@ -297,7 +297,7 @@ impl EventPoll { if !dst_file.supports_poll() { return Err(SystemError::ENOSYS); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let (ready_state, poll_epitems) = { let guard = if nonblock { diff --git a/kernel/src/filesystem/overlayfs/file.rs b/kernel/src/filesystem/overlayfs/file.rs index e91680124..d4e5b61ed 100644 --- a/kernel/src/filesystem/overlayfs/file.rs +++ b/kernel/src/filesystem/overlayfs/file.rs @@ -128,10 +128,7 @@ pub(super) fn write_at( // The legacy scalar inode ABI cannot retain a completed write while // unwinding for recovery. A backing sync charge is a terminal failure; // returning it as recoverable would repeat the already committed data. - result.sync_result.map_err(|error| match error { - crate::mm::allocation::MemoryAllocationError::System(error) => error, - crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + result.sync_result.map_err(|error| error.into_errno())?; Ok(result.written_len) } diff --git a/kernel/src/filesystem/overlayfs/metadata.rs b/kernel/src/filesystem/overlayfs/metadata.rs index 7af397948..67e356d14 100644 --- a/kernel/src/filesystem/overlayfs/metadata.rs +++ b/kernel/src/filesystem/overlayfs/metadata.rs @@ -349,10 +349,7 @@ impl XattrBuffer { /// backing errno, and report rejected scratch admission without reclaim/OOM /// under the caller's resource locks or overridden backing credentials. pub(super) fn xattr_read_error(error: MemoryAllocationError) -> SystemError { - match error { - MemoryAllocationError::System(error) => error, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - } + error.into_errno() } fn for_each_xattr_name>( diff --git a/kernel/src/filesystem/page_cache.rs b/kernel/src/filesystem/page_cache.rs index 3fd4664c4..9c98c9414 100644 --- a/kernel/src/filesystem/page_cache.rs +++ b/kernel/src/filesystem/page_cache.rs @@ -2081,7 +2081,7 @@ impl PageEntry { accounted_unevictable: AtomicBool::new(false), active_users: AtomicUsize::new(0), }) - .map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM))?; + .map_err(|_| MemoryAllocationError::Physical)?; Ok(Self { storage }) } @@ -2635,6 +2635,8 @@ impl PageCache { let mut pages = AccountedPageMap::new_in(HeapPlanAllocator::Auto); pages .try_reserve(required) + // Auto backing may fail its virtual-address reservation. Preserve + // that opaque allocation failure rather than guessing physical. .map_err(|_| SystemError::ENOMEM)?; let (pointer, layout) = pages.raw_table().allocation_info(); let charge = match unsafe { @@ -2658,9 +2660,12 @@ impl PageCache { HeapPlan::Physical.footprint(layout), )?; pages = AccountedPageMap::new_in(HeapPlanAllocator::Physical); - pages - .try_reserve(required) - .map_err(|_| SystemError::ENOMEM)?; + pages.try_reserve(required).map_err(|error| match error { + hashbrown::TryReserveError::CapacityOverflow => SystemError::ENOMEM.into(), + hashbrown::TryReserveError::AllocError { .. } => { + MemoryAllocationError::Physical + } + })?; let (direct_pointer, actual_layout) = pages.raw_table().allocation_info(); assert_eq!( actual_layout, layout, @@ -3196,10 +3201,7 @@ impl PageCache { } fn legacy_allocation_error(error: MemoryAllocationError) -> SystemError { - match error { - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - MemoryAllocationError::System(error) => error, - } + error.into_errno() } fn allocate_page_with_owner( @@ -3227,13 +3229,12 @@ impl PageCache { charge, ) } else { - crate::mm::page::allocate_registered_data_page( + crate::mm::page::allocate_registered_data_page_typed( page_type, self.page_flags(), Some(charge), ) } - .map_err(MemoryAllocationError::System) } fn prepare_entry( @@ -3242,9 +3243,8 @@ impl PageCache { state: PageState, creator: &Arc, ) -> Result, MemoryAllocationError> { - let result = PageEntry::try_new(page.clone(), state, creator).and_then(|entry| { - Arc::try_new(entry).map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM)) - }); + let result = PageEntry::try_new(page.clone(), state, creator) + .and_then(|entry| Arc::try_new(entry).map_err(|_| MemoryAllocationError::Physical)); if result.is_err() { self.discard_unlinked_page(page); } diff --git a/kernel/src/filesystem/page_cache/ordered_index.rs b/kernel/src/filesystem/page_cache/ordered_index.rs index cfe9c9929..99236e676 100644 --- a/kernel/src/filesystem/page_cache/ordered_index.rs +++ b/kernel/src/filesystem/page_cache/ordered_index.rs @@ -5,7 +5,6 @@ use crate::cgroup::memory::{accounting::KernelByteKind, object::ObjectAccount, o use crate::mm::allocation::MemoryAllocationError; use alloc::sync::Arc; use core::ops::{Bound, RangeBounds}; -use system_error::SystemError; #[derive(Debug)] pub(super) struct Node { @@ -27,7 +26,7 @@ impl Node { right: None, height: 1, }) - .map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM)) + .map_err(|_| MemoryAllocationError::Physical) } fn height(node: &Option>) -> i32 { diff --git a/kernel/src/filesystem/vfs/file.rs b/kernel/src/filesystem/vfs/file.rs index fb5eaf533..a884056b4 100644 --- a/kernel/src/filesystem/vfs/file.rs +++ b/kernel/src/filesystem/vfs/file.rs @@ -1070,9 +1070,12 @@ impl File { ) -> Result<(), MemoryAllocationError> { let inode = self.inode(); let sync_result = inode.sync_file_range(start, end, datasync, self.private_data.lock()); - // Charge is an internal retry, not the final fsync result. Preserve + // Recoverable allocation failure is not the final fsync result. Preserve // the OFD cursor until synchronization reaches a terminal result. - if matches!(&sync_result, Err(MemoryAllocationError::Charge(_))) { + if sync_result + .as_ref() + .is_err_and(|error| error.is_recoverable()) + { return sync_result; } let wb_result = match inode.page_cache() { @@ -1313,12 +1316,10 @@ impl File { } Ok(false) => {} Err(error) => { - // Data is already committed. Never let a metadata Charge - // escape to the primary-write recovery loop and replay it. - result.sync_result = Err(match error { - MemoryAllocationError::Charge(_) => SystemError::ENOMEM.into(), - error => error, - }); + // Data is already committed and there is no continuation + // for this metadata operation. A sync retry cannot finish + // it, and a primary-write retry would duplicate the data. + result.sync_result = Err(error.into_errno().into()); return; } } @@ -2607,7 +2608,7 @@ impl File { result.completion = None; result.deferred.clear(); } - Err(MemoryAllocationError::Charge(_)) => { + Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => { if let Some(completion) = result.completion.take() { if result.deferred.try_reserve(1).is_err() { result.sync_result = Err(SystemError::ENOMEM.into()); @@ -2740,7 +2741,9 @@ impl File { Ok(_) => {} // Speculative prefetch never kills a task for an optional page. // Required read_at admission is handled by the inode outer. - Err(crate::mm::allocation::MemoryAllocationError::Charge(_)) => return Ok(()), + Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => { + return Ok(()) + } Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { return Err(error) } @@ -2760,7 +2763,9 @@ impl File { &creator, ) { Ok(_) => {} - Err(crate::mm::allocation::MemoryAllocationError::Charge(_)) => return Ok(()), + Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => { + return Ok(()) + } Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { return Err(error) } diff --git a/kernel/src/filesystem/vfs/file/allocation.rs b/kernel/src/filesystem/vfs/file/allocation.rs index f0b0fcece..691e06306 100644 --- a/kernel/src/filesystem/vfs/file/allocation.rs +++ b/kernel/src/filesystem/vfs/file/allocation.rs @@ -17,7 +17,7 @@ impl PreparedFile { /// Guardless storage-only recovery. Call before instance reservations, /// inode open side effects and publication; this does not open a File. pub(crate) fn new_recovering() -> Result { - let mut state = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut state = crate::mm::allocation::AllocationRecoveryState::default(); loop { match Self::try_new() { Ok(storage) => return Ok(storage), diff --git a/kernel/src/filesystem/vfs/file/io.rs b/kernel/src/filesystem/vfs/file/io.rs index 5afdf12dc..fa908837d 100644 --- a/kernel/src/filesystem/vfs/file/io.rs +++ b/kernel/src/filesystem/vfs/file/io.rs @@ -2,7 +2,7 @@ //! use the typed single-attempt methods instead, preserving their transaction. use super::{DelegatedWriteResult, File, MemoryAllocationError, SystemError}; -use crate::cgroup::memory::runtime::ChargeRecoveryState; +use crate::mm::allocation::AllocationRecoveryState; impl File { pub(super) fn set_completed_sync_range( @@ -53,7 +53,7 @@ impl File { pub(crate) fn recover_io( mut attempt: impl FnMut() -> Result, ) -> Result { - let mut recovery = ChargeRecoveryState::default(); + let mut recovery = AllocationRecoveryState::default(); loop { match attempt() { Ok(value) => return Ok(value), @@ -68,7 +68,7 @@ impl File { &self, mut result: DelegatedWriteResult, ) -> Result { - let mut recovery = ChargeRecoveryState::default(); + let mut recovery = AllocationRecoveryState::default(); loop { match core::mem::replace(&mut result.sync_result, Ok(())) { Ok(()) => { @@ -95,8 +95,7 @@ impl File { self.complete_write_success(&mut result); Ok(result.written_len) } - Err(MemoryAllocationError::System(error)) => Err(error), - Err(MemoryAllocationError::Charge(_)) => Err(SystemError::ENOMEM), + Err(error) => Err(error.into_errno()), } } diff --git a/kernel/src/filesystem/vfs/file/selftest.rs b/kernel/src/filesystem/vfs/file/selftest.rs index aaf90ecf2..923ab6295 100644 --- a/kernel/src/filesystem/vfs/file/selftest.rs +++ b/kernel/src/filesystem/vfs/file/selftest.rs @@ -152,6 +152,7 @@ struct SyncProbeInode { syncs: AtomicUsize, valid: AtomicBool, terminal_error: bool, + physical_error: bool, synchronous_after_write: bool, } @@ -222,6 +223,9 @@ impl IndexNode for SyncProbeInode { ); } if attempt == 0 { + if self.physical_error { + return Err(MemoryAllocationError::Physical); + } // Generate an actual typed quota failure on an isolated account. // Restore its margin before returning: real recovery can then // retry safely without reclaiming or selecting any OOM victim. @@ -276,12 +280,14 @@ impl IndexNode for SyncProbeInode { } } -/// Post-commit Charge retries synchronization only, preserving the strongest +/// Post-commit Charge/Physical retries synchronization only, preserving the strongest /// intent, translated exact range, unpublished position and typed errseq. /// The owning deferred completions must also release every retained File. pub(crate) fn run_write_completion() -> Result { - let mut valid = true; - for (terminal_error, mapping_error, full_sync, synchronous_after_write) in [ + let mut valid = !MemoryAllocationError::System(SystemError::ENOMEM).is_recoverable() + && MemoryAllocationError::Physical.is_recoverable() + && MemoryAllocationError::Physical.into_errno() == SystemError::ENOMEM; + for (physical_error, (terminal_error, mapping_error, full_sync, synchronous_after_write)) in [ (false, false, true, false), (true, false, true, false), (false, true, true, false), @@ -289,7 +295,10 @@ pub(crate) fn run_write_completion() -> Result { // Neither OFD requests sync initially. Only the leaf's post-write // mount-policy resample can discover Data and retain its retry range. (false, false, false, true), - ] { + ] + .into_iter() + .flat_map(|case| [false, true].map(|physical| (physical, case))) + { // Root accounts intentionally bypass controller limits. Use an // isolated child so max=0 really produces a recoverable quota error. let root = MemoryAccount::new(0x776602, None)?; @@ -306,6 +315,7 @@ pub(crate) fn run_write_completion() -> Result { syncs: AtomicUsize::new(0), valid: AtomicBool::new(true), terminal_error, + physical_error, synchronous_after_write, }) .map_err(|_| SystemError::ENOMEM)?; @@ -333,6 +343,7 @@ pub(crate) fn run_write_completion() -> Result { syncs: AtomicUsize::new(0), valid: AtomicBool::new(true), terminal_error: false, + physical_error: false, synchronous_after_write: false, }) .map_err(|_| SystemError::ENOMEM)?; @@ -370,7 +381,7 @@ pub(crate) fn run_write_completion() -> Result { && root.current_pages() == 0; if !outcome_valid || !progress_valid { log::error!( - "File completion selftest: terminal={terminal_error} mapping={mapping_error} \ + "File completion selftest: physical={physical_error} terminal={terminal_error} mapping={mapping_error} \ full={full_sync} resample={synchronous_after_write} outcome={outcome:?} \ pos={} writes={}/{} syncs={}/{} ranges={}/{}", outer.pos(), diff --git a/kernel/src/filesystem/vfs/open.rs b/kernel/src/filesystem/vfs/open.rs index 635093885..ec4a376d1 100644 --- a/kernel/src/filesystem/vfs/open.rs +++ b/kernel/src/filesystem/vfs/open.rs @@ -422,7 +422,7 @@ pub(crate) fn do_sys_openat2(dirfd: i32, path: &str, how: OpenHow) -> Result break storage, @@ -870,7 +870,7 @@ pub fn do_open_execat_with_flags( super::permission::check_inode_permission(&inode, &metadata, PermissionMask::MAY_EXEC)?; // 创建File对象,使用O_RDONLY | O_CLOEXEC - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); let file = loop { match File::new(inode.clone(), FileFlags::O_RDONLY | FileFlags::O_CLOEXEC) { Ok(file) => break file, diff --git a/kernel/src/filesystem/vfs/syscall/sys_fallocate.rs b/kernel/src/filesystem/vfs/syscall/sys_fallocate.rs index e71c596ff..dd56c57b4 100644 --- a/kernel/src/filesystem/vfs/syscall/sys_fallocate.rs +++ b/kernel/src/filesystem/vfs/syscall/sys_fallocate.rs @@ -51,7 +51,7 @@ impl Syscall for SysFallocateHandle { if let Some(file) = fd_table_guard.get_file_by_fd(fd) { drop(fd_table_guard); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); loop { match crate::filesystem::vfs::vcore::vfs_fallocate_file( file.clone(), diff --git a/kernel/src/filesystem/vfs/syscall/sys_ioctl.rs b/kernel/src/filesystem/vfs/syscall/sys_ioctl.rs index d4c47a457..c943f3d96 100644 --- a/kernel/src/filesystem/vfs/syscall/sys_ioctl.rs +++ b/kernel/src/filesystem/vfs/syscall/sys_ioctl.rs @@ -89,7 +89,7 @@ impl Syscall for SysIoctlHandle { } _ => { // 其他命令转发给inode处理 - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); loop { // The single attempt owns the private guard. It is gone // before recovery, and drivers admit resources before diff --git a/kernel/src/filesystem/vfs/syscall/sys_memfd_create.rs b/kernel/src/filesystem/vfs/syscall/sys_memfd_create.rs index 5e21abdb2..3d7d0bdf8 100644 --- a/kernel/src/filesystem/vfs/syscall/sys_memfd_create.rs +++ b/kernel/src/filesystem/vfs/syscall/sys_memfd_create.rs @@ -62,7 +62,7 @@ impl Syscall for SysMemfdCreate { return Err(SystemError::ENOSYS); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let file = loop { let mut attempt_name = Vec::new(); attempt_name diff --git a/kernel/src/filesystem/vfs/syscall/sys_truncate.rs b/kernel/src/filesystem/vfs/syscall/sys_truncate.rs index 4e57aed57..118a55d57 100644 --- a/kernel/src/filesystem/vfs/syscall/sys_truncate.rs +++ b/kernel/src/filesystem/vfs/syscall/sys_truncate.rs @@ -69,7 +69,7 @@ impl Syscall for SysTruncateHandle { } } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); while let Err(error) = vfs_truncate(target.clone(), length) { error.recover(&mut recovery)?; } diff --git a/kernel/src/init/initram.rs b/kernel/src/init/initram.rs index e0b0b2a20..7e36ce16d 100644 --- a/kernel/src/init/initram.rs +++ b/kernel/src/init/initram.rs @@ -97,10 +97,7 @@ struct CpioEntryInfo { pub fn initramfs_init() -> Result<(), SystemError> { // The initcall ABI is a terminal bootstrap boundary, not a task-context // reclaim/retry site. Keep admission errors typed throughout extraction. - initramfs_init_typed().map_err(|error| match error { - crate::mm::allocation::MemoryAllocationError::System(error) => error, - crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - }) + initramfs_init_typed().map_err(|error| error.into_errno()) } fn initramfs_init_typed() -> Result<(), crate::mm::allocation::MemoryAllocationError> { diff --git a/kernel/src/ipc/syscall/sys_shmat.rs b/kernel/src/ipc/syscall/sys_shmat.rs index 7f0e45e03..98346d59e 100644 --- a/kernel/src/ipc/syscall/sys_shmat.rs +++ b/kernel/src/ipc/syscall/sys_shmat.rs @@ -79,7 +79,7 @@ pub(super) fn do_kernel_shmat( let readonly = shmflg.contains(ShmFlags::SHM_RDONLY); // attach_guard pins the exact segment; attach_begin's manager guard was // released above. Retrying private File birth must not relookup a reused ID. - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let sysv_attach = loop { match attach_guard.create_attach(readonly) { Ok(attach) => break attach, diff --git a/kernel/src/ipc/syscall/sys_shmget.rs b/kernel/src/ipc/syscall/sys_shmget.rs index 9b8a9a848..2255b754d 100644 --- a/kernel/src/ipc/syscall/sys_shmget.rs +++ b/kernel/src/ipc/syscall/sys_shmget.rs @@ -54,7 +54,7 @@ pub(super) fn do_kernel_shmget( } let ipcns = ProcessManager::current_ipcns(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); match key { IPC_PRIVATE => { diff --git a/kernel/src/libs/elf/scratch_selftest.rs b/kernel/src/libs/elf/scratch_selftest.rs index d5596feb3..0a8baec4f 100644 --- a/kernel/src/libs/elf/scratch_selftest.rs +++ b/kernel/src/libs/elf/scratch_selftest.rs @@ -9,7 +9,7 @@ use crate::{ FileSystem, FileType, InodeMode, }, }, - mm::kernel_buffer::KernelBuffer, + mm::{allocation::MemoryAllocationError, kernel_buffer::KernelBuffer}, }; pub(crate) fn run() -> bool { @@ -31,18 +31,21 @@ pub(crate) fn run() -> bool { ElfLoader::FILE_READ_BUF_SIZE + 1, 0, &file, - |size| { + |size| -> Result { requested = size; - File::recover_io(|| { - KernelBuffer::try_new_zeroed_in_with( - size, - prepared.object_account(), - &mut || { - backing_attempts += 1; - Err(SystemError::ENOMEM) - }, - ) - }) + // This fixture checks a single refused scratch birth before + // population, independently of global recovery policy. + match KernelBuffer::try_new_zeroed_in_with( + size, + prepared.object_account(), + &mut || { + backing_attempts += 1; + Err(SystemError::ENOMEM) + }, + ) { + Err(error @ MemoryAllocationError::Physical) => Err(error.into_errno()), + _ => Err(SystemError::EINVAL), + } }, ); Ok(matches!(result, Err(SystemError::ENOMEM)) diff --git a/kernel/src/libs/futex/futex.rs b/kernel/src/libs/futex/futex.rs index 4a41566a0..ed0fcdea8 100644 --- a/kernel/src/libs/futex/futex.rs +++ b/kernel/src/libs/futex/futex.rs @@ -395,7 +395,7 @@ impl Futex { ); let current_address_space = AddressSpace::current()?; let mut flags = FaultFlags::FAULT_FLAG_WRITE; - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); @@ -459,6 +459,17 @@ impl Futex { )?; continue; } + Err(error @ crate::mm::allocation::MemoryAllocationError::Physical) => { + if let Err(error) = error.recover(&mut charge_recovery) { + crate::mm::fault::account_fault_recovery_failure( + &data_account, + flags, + error.clone(), + ); + return Err(error); + } + continue; + } Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { Err(error) } diff --git a/kernel/src/mm/allocation.rs b/kernel/src/mm/allocation.rs index edf0fd023..0c33fa393 100644 --- a/kernel/src/mm/allocation.rs +++ b/kernel/src/mm/allocation.rs @@ -7,9 +7,24 @@ use crate::cgroup::memory::{ runtime::{recover_charge, ChargeRecoveryState, MemoryAllocPolicy, RecoveryDecision}, }; use crate::mm::oom_reaper::JobBirthError; +use crate::{ + arch::CurrentIrqArch, + exception::{in_interrupt, InterruptArch}, + process::ProcessManager, +}; + +/// Common eligibility for sleepable allocation work, independent of the VA backend. +pub(crate) fn task_context() -> bool { + CurrentIrqArch::is_irq_enabled() + && !in_interrupt() + && ProcessManager::initialized() + && ProcessManager::current_pcb().preempt_count() == 0 +} #[derive(Debug)] pub(crate) enum MemoryAllocationError { + /// A real frame or fixed-layout heap allocation failed. Never infer this from errno. + Physical, Charge(ChargeFailure), System(SystemError), } @@ -30,17 +45,75 @@ impl From for MemoryAllocationError { fn from(error: JobBirthError) -> Self { match error { JobBirthError::Charge(failure) => Self::Charge(failure), - JobBirthError::Physical(_) => Self::System(SystemError::ENOMEM), + JobBirthError::Physical(_) => Self::Physical, } } } +/// One resource-lock-free outer transaction. Quota and global pressure have +/// distinct accounting and budgets; unrelated syscalls never share this state. +pub(crate) struct AllocationRecoveryState { + charge: ChargeRecoveryState, + global: super::fault_recovery::GlobalFaultRecoveryState, + physical_deadline: Option, +} + +impl AllocationRecoveryState { + pub(crate) const fn new() -> Self { + Self { + charge: ChargeRecoveryState::new(), + global: super::fault_recovery::GlobalFaultRecoveryState::new(), + physical_deadline: None, + } + } + + fn recover_physical(&mut self) -> Result<(), SystemError> { + // Check task/IRQ/preemption before reading any task-local NOIO state. + if !task_context() || crate::cgroup::memory::io::noio_active() { + return Err(SystemError::ENOMEM); + } + let deadline = *self + .physical_deadline + .get_or_insert_with(|| crate::time::timer::next_n_us_timer_jiffies(10_000_000)); + if crate::time::timer::clock() >= deadline { + return Err(SystemError::ENOMEM); + } + // Reuse fault reclaim's bounded scan and killable writeback waits, but + // keep a fixed transaction deadline even when unrelated pages release. + let retry = self.global.retry()?; + if retry && crate::time::timer::clock() < deadline { + Ok(()) + } else { + Err(SystemError::ENOMEM) + } + } +} + +impl Default for AllocationRecoveryState { + fn default() -> Self { + Self::new() + } +} + impl MemoryAllocationError { + pub(crate) fn is_recoverable(&self) -> bool { + matches!(self, Self::Charge(_) | Self::Physical) + } + + /// Terminal/legacy boundary; an errno never grants recovery authority. + pub(crate) fn into_errno(self) -> SystemError { + match self { + Self::System(error) => error, + Self::Physical | Self::Charge(_) => SystemError::ENOMEM, + } + } + /// Call only after releasing resource guards and failed MM owners. /// Ok means retry the complete unpublished operation with a fresh creator. - pub(crate) fn recover(self, state: &mut ChargeRecoveryState) -> Result<(), SystemError> { + pub(crate) fn recover(self, state: &mut AllocationRecoveryState) -> Result<(), SystemError> { match self { Self::System(error) => Err(error), + Self::Physical => state.recover_physical(), // The hard counter cap is never reclaimable, including during // bootstrap before a current task exists. Do not query NOIO first. Self::Charge(failure) if failure.reason == ChargeFailureReason::CounterOverflow => { @@ -52,7 +125,7 @@ impl MemoryAllocationError { } else { MemoryAllocPolicy::ReclaimThenOom }; - match recover_charge(&failure, policy, 0, state)? { + match recover_charge(&failure, policy, 0, &mut state.charge)? { RecoveryDecision::RetryOperation => Ok(()), RecoveryDecision::Exhausted => Err(SystemError::ENOMEM), } @@ -60,3 +133,33 @@ impl MemoryAllocationError { } } } + +#[cfg(feature = "mm_selftest")] +pub(crate) fn recovery_selftest() -> bool { + if !task_context() { + return false; + } + let mut state = AllocationRecoveryState::new(); + // An errno alone must never enter reclaim or start its budget. + for error in [SystemError::ENOMEM, SystemError::EIO, SystemError::EFAULT] { + if MemoryAllocationError::System(error.clone()).is_recoverable() + || MemoryAllocationError::System(error.clone()).recover(&mut state) != Err(error) + || state.physical_deadline.is_some() + { + return false; + } + } + { + let _noio = crate::cgroup::memory::io::MemoryIoScope::enter_noio(); + if MemoryAllocationError::Physical.recover(&mut state) != Err(SystemError::ENOMEM) + || state.physical_deadline.is_some() + { + return false; + } + } + state.physical_deadline = Some(crate::time::timer::clock()); + MemoryAllocationError::Physical.recover(&mut state) == Err(SystemError::ENOMEM) + && MemoryAllocationError::Physical.is_recoverable() + && MemoryAllocationError::Physical.into_errno() == SystemError::ENOMEM + && crate::driver::block::loop_device::LoopDevice::physical_boundary_selftest() +} diff --git a/kernel/src/mm/allocator/kernel_allocator.rs b/kernel/src/mm/allocator/kernel_allocator.rs index 1826ece5c..c3f909bf0 100644 --- a/kernel/src/mm/allocator/kernel_allocator.rs +++ b/kernel/src/mm/allocator/kernel_allocator.rs @@ -55,6 +55,13 @@ pub(super) fn large_allocation_bytes() -> u64 { } impl KernelAllocator { + /// Structural limit of the direct backend, independent of current pressure. + pub(crate) fn supports_physical_layout(layout: Layout) -> bool { + !super::footprint::uses_buddy(layout) + || super::footprint::buddy_frame_count(layout) + <= (1usize << (super::buddy::BUDDY_ORDER_COUNT - 1)) + } + unsafe fn alloc_in_buddy( &self, layout: Layout, diff --git a/kernel/src/mm/cpu_buffer.rs b/kernel/src/mm/cpu_buffer.rs index ce055f23b..e3ca2b3f0 100644 --- a/kernel/src/mm/cpu_buffer.rs +++ b/kernel/src/mm/cpu_buffer.rs @@ -90,11 +90,8 @@ impl CpuBuffer { #[cfg(feature = "mm_selftest")] retired: None, })?; - let buffer = - KernelBuffer::try_new_zeroed_virtual(capacity).map_err(|error| match error { - super::allocation::MemoryAllocationError::System(error) => error, - super::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - })?; + let buffer = KernelBuffer::try_new_zeroed_virtual(capacity) + .map_err(|error| error.into_errno())?; node.value_mut().buffer = Some(buffer); return Ok(Self { backend: Backend::Mapped(Some(node)), diff --git a/kernel/src/mm/fault.rs b/kernel/src/mm/fault.rs index f0c10e0cb..ce149fc9a 100644 --- a/kernel/src/mm/fault.rs +++ b/kernel/src/mm/fault.rs @@ -82,7 +82,7 @@ pub(crate) fn account_fault_result( /// PGFAULT, but an interrupted incomplete fault is not an OOM/minor/major fault. pub(crate) fn recover_fault_charge( failure: ChargeFailure, - state: &mut crate::cgroup::memory::runtime::ChargeRecoveryState, + state: &mut crate::mm::allocation::AllocationRecoveryState, account: &MemoryAccount, flags: FaultFlags, ) -> Result<(), SystemError> { @@ -343,9 +343,8 @@ impl<'a> PageFaultMessage<'a> { crate::mm::allocation::MemoryAllocationError::Charge(failure) => { self.fail_charge(failure) } - crate::mm::allocation::MemoryAllocationError::System(SystemError::ENOMEM) => { - VmFaultReason::VM_FAULT_OOM - } + crate::mm::allocation::MemoryAllocationError::System(SystemError::ENOMEM) + | crate::mm::allocation::MemoryAllocationError::Physical => VmFaultReason::VM_FAULT_OOM, crate::mm::allocation::MemoryAllocationError::System(_) => { VmFaultReason::VM_FAULT_SIGBUS } diff --git a/kernel/src/mm/fault_recovery.rs b/kernel/src/mm/fault_recovery.rs index de1a2edec..bf33477b2 100644 --- a/kernel/src/mm/fault_recovery.rs +++ b/kernel/src/mm/fault_recovery.rs @@ -1,4 +1,5 @@ -//! Physical allocation recovery at the resource-lock-free user fault boundary. +//! Shared physical allocation reclaim at resource-lock-free transaction boundaries. +//! User faults own their OOM handoff; fallible operations own a fixed total deadline. //! Charge/memcg failures retain their separate policy and limiting account. use core::sync::atomic::{AtomicUsize, Ordering}; diff --git a/kernel/src/mm/kernel_buffer.rs b/kernel/src/mm/kernel_buffer.rs index 6177c4bf0..7f28484ce 100644 --- a/kernel/src/mm/kernel_buffer.rs +++ b/kernel/src/mm/kernel_buffer.rs @@ -54,7 +54,7 @@ impl KernelBuffer { /// Prepare local scratch at a resource-lock-free task boundary. Only the /// unpublished allocation is retried; callers must perform I/O afterwards. pub(crate) fn new_zeroed_recovering(len: usize) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); loop { match Self::try_new_zeroed(len) { Ok(buffer) => return Ok(buffer), @@ -188,6 +188,11 @@ impl KernelBuffer { // high-order pressure and rounding waste. Otherwise retain the heap // backend for all sizes, with the same admission and no-retry policy. if heap_supported && (len <= 2 * MMArch::PAGE_SIZE || !virtual_supported) { + if !crate::mm::allocator::kernel_allocator::KernelAllocator::supports_physical_layout( + layout, + ) { + return Err(SystemError::ENOMEM.into()); + } let charge = owner.try_charge_allocation(KernelByteKind::SlabUnreclaimable, layout)?; #[cfg(feature = "mm_selftest")] let pointer = if fail_heap { @@ -210,7 +215,7 @@ impl KernelBuffer { // above is propagated, not converted into a fallback opportunity. drop(charge); if len <= MMArch::PAGE_SIZE || !virtual_supported { - return Err(SystemError::ENOMEM.into()); + return Err(MemoryAllocationError::Physical); } } #[cfg(feature = "mm_selftest")] diff --git a/kernel/src/mm/kernel_buffer/selftest.rs b/kernel/src/mm/kernel_buffer/selftest.rs index 8f32b75d6..65daa1f72 100644 --- a/kernel/src/mm/kernel_buffer/selftest.rs +++ b/kernel/src/mm/kernel_buffer/selftest.rs @@ -121,10 +121,7 @@ fn failure_policy() -> bool { calls += 1; Err(SystemError::ENOMEM.into()) }); - let rejected = matches!( - &result, - Err(MemoryAllocationError::System(SystemError::ENOMEM)) - ); + let rejected = matches!(&result, Err(MemoryAllocationError::Physical)); drop(result); if !rejected || calls != 0 || !empty(&account) { return false; @@ -235,15 +232,20 @@ fn unsupported_virtual() -> bool { return false; } let result = KernelBuffer::allocate_in(len, owner, false, true, &mut unexpected); - if !matches!( - result, - Err(MemoryAllocationError::System(SystemError::ENOMEM)) - ) || !empty(&account) - { + if !matches!(result, Err(MemoryAllocationError::Physical)) || !empty(&account) { return false; } } - // Even a valid Layout too large for buddy must fail without leaking charge. + // A valid Layout can exceed the backend's structural capacity. Reclaim + // cannot satisfy it, and it must never acquire a charge or retry authority. + let supported = + crate::mm::allocator::kernel_allocator::KernelAllocator::supports_physical_layout; + if !supported(Layout::from_size_align(1usize << 30, 1).unwrap()) + || supported(Layout::from_size_align((1usize << 30) + 1, 1).unwrap()) + || supported(Layout::from_size_align(1usize << 32, 1).unwrap()) + { + return false; + } let result = KernelBuffer::allocate_in(1usize << 32, owner, false, false, &mut unexpected); if !matches!( result, @@ -506,6 +508,8 @@ fn typed_failure_rollback() -> bool { return false; }; let owner = prepared.object_account(); + // An injected data-frame ENOMEM represents a real Physical failure, + // unlike a generic errno returned by the virtual backend itself. // Fail every data-frame boundary in a four-page typed slice. Successfully // donated frames must be unmapped/freed before the charge is released. for fail_after in 0..4 { @@ -530,10 +534,7 @@ fn typed_failure_rollback() -> bool { }, ) .map(|storage| KernelSlice::initialize(storage, 1537, 9usize)); - let failed = matches!( - &result, - Err(MemoryAllocationError::System(SystemError::ENOMEM)) - ); + let failed = matches!(&result, Err(MemoryAllocationError::Physical)); drop(result); failed && calls == fail_after + 1 && storage_released(&account) }); diff --git a/kernel/src/mm/page.rs b/kernel/src/mm/page.rs index f0102c755..c0bef25ad 100644 --- a/kernel/src/mm/page.rs +++ b/kernel/src/mm/page.rs @@ -10,6 +10,7 @@ use core::{ use system_error::SystemError; use unified_init::macros::unified_init; +use super::allocation::MemoryAllocationError; use super::page_lru::{PageLru, PreparedPageLruCapacity}; pub(crate) use super::page_rmap::PageMapping; use super::page_rmap::PageVmaSet; @@ -457,7 +458,25 @@ pub(crate) fn allocate_registered_data_page( flags: PageFlags, charge: Option, ) -> Result, SystemError> { - allocate_registered_data_page_initialized(page_type, flags, charge, |_| Ok(())) + allocate_registered_data_page_typed(page_type, flags, charge) + .map_err(MemoryAllocationError::into_errno) +} + +/// Preserve physical pressure until the cache caller reaches its lock-free +/// recovery boundary. Legacy fault callers retain their original errno API. +pub(crate) fn allocate_registered_data_page_typed( + page_type: PageType, + flags: PageFlags, + charge: Option, +) -> Result, MemoryAllocationError> { + let request = data_frame_request(PageFrameCount::ONE, charge.as_ref()); + allocate_registered_data_page_initialized_with_request( + page_type, + flags, + charge, + |_| Ok(()), + request, + ) } /// Copy before publication: the source Arc pins its frame without a registry @@ -494,6 +513,7 @@ pub(crate) fn allocate_registered_data_page_initialized( allocate_registered_data_page_initialized_with_request( page_type, flags, charge, initialize, request, ) + .map_err(MemoryAllocationError::into_errno) } fn reclaim_io_backing( @@ -537,6 +557,7 @@ pub(crate) fn reclaim_io_admission_selftest() -> bool { && !reclaim_io_backing(MemoryKind::File, true, ReserveLevel::OomExit) && scan_visit_budget_selftest() && crate::mm::fault_recovery::budget_selftest() + && crate::mm::allocation::recovery_selftest() } /// Only PageCache's File path may identify a necessary stacked-I/O backing @@ -546,9 +567,9 @@ pub(crate) fn allocate_registered_file_cache_page( page_type: PageType, flags: PageFlags, charge: MemoryCharge, -) -> Result, SystemError> { +) -> Result, MemoryAllocationError> { if !matches!(&page_type, PageType::File(_)) || charge.kind() != MemoryKind::File { - return Err(SystemError::EINVAL); + return Err(SystemError::EINVAL.into()); } let request = if reclaim_io_backing( charge.kind(), @@ -590,12 +611,12 @@ fn allocate_registered_data_page_initialized_with_request( charge: Option, initialize: impl FnOnce(PhysAddr) -> Result<(), SystemError>, request: FrameAllocationRequest, -) -> Result, SystemError> { +) -> Result, MemoryAllocationError> { if charge .as_ref() .is_some_and(|charge| charge.pages() != 1 || !page_type.accepts_memory_kind(charge.kind())) { - return Err(SystemError::EINVAL); + return Err(SystemError::EINVAL.into()); } // Only this registered single-page path has the audited data lifecycle. // Raw/contiguous/legacy-manager callers keep their original default. This @@ -612,7 +633,7 @@ fn allocate_registered_data_page_initialized_with_request( LockedFrameAllocator .allocate_request(request) .map(|(paddr, _)| paddr) - .ok_or(SystemError::ENOMEM)? + .ok_or(MemoryAllocationError::Physical)? }; let mut raw = RawFrameExtent { next: paddr, @@ -623,7 +644,7 @@ fn allocate_registered_data_page_initialized_with_request( initialize(paddr)?; // The constructor takes ownership on success AND Arc allocation failure. raw.remaining = 0; - let page = Page::try_new_with_charge(paddr, page_type, flags, charge)?; + let page = Page::try_new_with_charge_typed(paddr, page_type, flags, charge)?; loop { let mut manager_capacity = prepare_manager_capacity()?; let mut lru_capacity = if flags.contains(PageFlags::PG_LRU) { @@ -695,7 +716,7 @@ fn allocate_registered_data_page_initialized_with_request( drop(manager_capacity); match result { Some(Ok(())) => return Ok(page), - Some(Err(error)) => return Err(error), + Some(Err(error)) => return Err(error.into()), None => {} } } @@ -2079,6 +2100,16 @@ impl Page { flags: PageFlags, memory_charge: Option, ) -> Result, SystemError> { + Self::try_new_with_charge_typed(phys_addr, page_type, flags, memory_charge) + .map_err(MemoryAllocationError::into_errno) + } + + fn try_new_with_charge_typed( + phys_addr: PhysAddr, + page_type: PageType, + flags: PageFlags, + memory_charge: Option, + ) -> Result, MemoryAllocationError> { let inner = InnerPage::new( phys_addr, page_type, @@ -2095,7 +2126,7 @@ impl Page { contiguous_owner: None, memory_charge, }) - .map_err(|_| SystemError::ENOMEM)?; + .map_err(|_| MemoryAllocationError::Physical)?; // The unlocked birth API publishes manager/LRU together. Construction // alone must not expose a page through either registry. Ok(page) diff --git a/kernel/src/mm/page/migration/selftest/shared.rs b/kernel/src/mm/page/migration/selftest/shared.rs index dad768da7..ef3b8b513 100644 --- a/kernel/src/mm/page/migration/selftest/shared.rs +++ b/kernel/src/mm/page/migration/selftest/shared.rs @@ -28,10 +28,8 @@ impl CacheFixture { (None, None) } else { let fs = crate::filesystem::ramfs::RamFS::new(); - let fixture_error = |error| match error { - crate::mm::allocation::MemoryAllocationError::System(error) => error, - crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - }; + let fixture_error = + |error: crate::mm::allocation::MemoryAllocationError| error.into_errno(); let inode = fs .root_inode() .create( diff --git a/kernel/src/mm/remote_access.rs b/kernel/src/mm/remote_access.rs index b89889dee..ba1e72404 100644 --- a/kernel/src/mm/remote_access.rs +++ b/kernel/src/mm/remote_access.rs @@ -434,7 +434,7 @@ fn fault_in_remote_page( force: bool, ) -> Result { let mut tried = false; - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&crate::process::ProcessManager::current_pcb()); @@ -574,6 +574,18 @@ fn fault_in_remote_page( continue; } Err(crate::mm::allocation::MemoryAllocationError::System(error)) => Err(error), + Err(error @ crate::mm::allocation::MemoryAllocationError::Physical) => { + if let Err(error) = error.recover(&mut charge_recovery) { + crate::mm::fault::account_fault_recovery_failure( + &data_account, + retry_flags, + error.clone(), + ); + return Err(error); + } + tried = true; + continue; + } Ok(()) => Ok(()), }; if let Err(error) = result { diff --git a/kernel/src/mm/syscall/sys_fadvise64.rs b/kernel/src/mm/syscall/sys_fadvise64.rs index a05364c14..35736e5d8 100644 --- a/kernel/src/mm/syscall/sys_fadvise64.rs +++ b/kernel/src/mm/syscall/sys_fadvise64.rs @@ -181,7 +181,10 @@ pub(crate) fn generic_fadvise( Ok(_) => file.set_ra_state(ra_state)?, // Linux WILLNEED is advisory: failure to prefetch does not // force OOM or make the subsequent required read succeed. - Err(crate::mm::allocation::MemoryAllocationError::Charge(_)) => {} + Err( + crate::mm::allocation::MemoryAllocationError::Charge(_) + | crate::mm::allocation::MemoryAllocationError::Physical, + ) => {} Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { return Err(error) } diff --git a/kernel/src/mm/syscall/sys_mlock.rs b/kernel/src/mm/syscall/sys_mlock.rs index 8d8977703..f43ed4cd7 100644 --- a/kernel/src/mm/syscall/sys_mlock.rs +++ b/kernel/src/mm/syscall/sys_mlock.rs @@ -64,7 +64,7 @@ pub(super) fn do_mlock( let vm = AddressSpace::current()?; let region = VirtRegion::new(start, len); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&crate::process::ProcessManager::current_pcb()); diff --git a/kernel/src/mm/syscall/sys_mlock2.rs b/kernel/src/mm/syscall/sys_mlock2.rs index 5cd9487e7..7f439ffb5 100644 --- a/kernel/src/mm/syscall/sys_mlock2.rs +++ b/kernel/src/mm/syscall/sys_mlock2.rs @@ -74,7 +74,7 @@ fn do_mlock2(start: VirtAddr, len: usize, flags: usize) -> Result Err(error), + Err(error @ crate::mm::allocation::MemoryAllocationError::Physical) => { + if let Err(error) = error.recover(&mut charge_recovery) { + crate::mm::fault::account_fault_recovery_failure( + &data_account, + retry_flags, + error.clone(), + ); + return Err(error); + } + retried = true; + continue; + } Ok(()) => Ok(()), }; if let Err(error) = result { @@ -546,7 +558,7 @@ impl AddressSpace { user_ns: Arc, dumpable: u8, ) -> Result, SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match Self::new_for_task_with_creator( @@ -689,6 +701,7 @@ impl AddressSpace { ) { Ok(mm) => mm, Err(MemoryAllocationError::System(error)) => return Err(error), + Err(MemoryAllocationError::Physical) => return Err(SystemError::ENOMEM), // An unlimited private fixture must not reject admission. This is // a failed assertion, not production scope-to-errno conversion. Err(MemoryAllocationError::Charge(_)) => return Ok(false), @@ -1329,7 +1342,7 @@ impl AddressSpace { round_to_min: bool, allocate_at_once: bool, ) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); let len = page_align_up(len); if map_flags.contains(MapFlags::MAP_SHARED) { if len == 0 { @@ -1524,7 +1537,7 @@ impl AddressSpace { self: &Arc, args: FileMappingWithFileArgs, ) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); let FileMappingWithFileArgs { file, start_vaddr, @@ -2046,7 +2059,7 @@ impl AddressSpace { page_count: PageFrameCount, ) -> Result<(), SystemError> { let region = VirtRegion::new(start_page.virt_address(), page_count.bytes()); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let mut guard = self.write(); @@ -2103,7 +2116,7 @@ impl AddressSpace { prot_flags: ProtFlags, ) -> Result<(), SystemError> { let region = VirtRegion::new(start_page.virt_address(), page_count.bytes()); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2167,7 +2180,7 @@ impl AddressSpace { behavior: MadvFlags, ) -> Result<(), SystemError> { let region = VirtRegion::new(start_page.virt_address(), page_count.bytes()); - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2251,7 +2264,7 @@ impl AddressSpace { mremap_flags: MremapFlags, new_vaddr: VirtAddr, ) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2459,7 +2472,7 @@ impl AddressSpace { } pub fn set_brk_wait(self: &Arc, new_addr: VirtAddr) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2537,7 +2550,7 @@ impl AddressSpace { } pub fn sbrk_wait(self: &Arc, incr: isize) -> Result { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); let data_account = self.memory_account(); @@ -2608,7 +2621,7 @@ impl AddressSpace { self: &Arc, owner: &Arc, ) -> Result, SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { match self.try_clone_for_task(owner) { Ok(child) => return Ok(child), diff --git a/kernel/src/mm/ucontext/uprobe/reconcile.rs b/kernel/src/mm/ucontext/uprobe/reconcile.rs index 82a5a8703..17411504f 100644 --- a/kernel/src/mm/ucontext/uprobe/reconcile.rs +++ b/kernel/src/mm/ucontext/uprobe/reconcile.rs @@ -538,7 +538,8 @@ fn uprobe_register_best_effort_once( crate::mm::allocation::MemoryAllocationError::System(error), )) => Err(error), Err(LockedRegisterError::Allocation( - crate::mm::allocation::MemoryAllocationError::Charge(_), + crate::mm::allocation::MemoryAllocationError::Charge(_) + | crate::mm::allocation::MemoryAllocationError::Physical, )) => Err(SystemError::EAGAIN_OR_EWOULDBLOCK), } } diff --git a/kernel/src/mm/ucontext/uprobe/site.rs b/kernel/src/mm/ucontext/uprobe/site.rs index f7c473f4c..b3f8f4dab 100644 --- a/kernel/src/mm/ucontext/uprobe/site.rs +++ b/kernel/src/mm/ucontext/uprobe/site.rs @@ -541,7 +541,7 @@ pub(super) fn uprobe_register( consumer: &Arc, expected_mapping: &ExpectedProbeMapping, ) -> Result, SystemError> { - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); let result = loop { let creator = crate::cgroup::memory::object_for_task(&crate::process::ProcessManager::current_pcb()); diff --git a/kernel/src/mm/ucontext/vma_ops.rs b/kernel/src/mm/ucontext/vma_ops.rs index 0a64118a8..5024e9651 100644 --- a/kernel/src/mm/ucontext/vma_ops.rs +++ b/kernel/src/mm/ucontext/vma_ops.rs @@ -725,10 +725,7 @@ impl InnerAddressSpace { notifications: rollback_notifications, }); } - terminal_error = Some(match err { - MemoryAllocationError::System(err) => err, - MemoryAllocationError::Charge(_) => SystemError::ENOMEM, - }); + terminal_error = Some(err.into_errno()); break; } }; diff --git a/kernel/src/mm/vmalloc.rs b/kernel/src/mm/vmalloc.rs index d5f99c09a..947eb9a42 100644 --- a/kernel/src/mm/vmalloc.rs +++ b/kernel/src/mm/vmalloc.rs @@ -4,6 +4,7 @@ //! uses only non-recursive local metadata storage; interrupt-side destruction //! transfers its existing record to the permanent memory notification worker. +pub(crate) use super::allocation::task_context; #[cfg(feature = "mm_selftest")] use crate::cgroup::memory::object_for_task; use crate::{ @@ -12,7 +13,7 @@ use crate::{ accounting::KernelByteKind, object::{ObjectAccount, ObjectCharge}, }, - exception::{in_interrupt, InterruptArch}, + exception::InterruptArch, libs::{ rbtree::{PreparedNode, RBTree}, spinlock::SpinLock, @@ -79,12 +80,12 @@ pub(crate) fn overlaps(start: VirtAddr, end: VirtAddr) -> bool { start < end && start.data() < BASE + SIZE && end.data() > BASE } -unsafe fn metadata_box(value: T) -> Result, SystemError> { +unsafe fn metadata_box(value: T) -> Result, MemoryAllocationError> { let pointer = KERNEL_ALLOCATOR .alloc_metadata(Layout::new::()) .cast::(); if pointer.is_null() { - return Err(SystemError::ENOMEM); + return Err(MemoryAllocationError::Physical); } pointer.write(value); Ok(Box::from_raw(pointer)) @@ -294,13 +295,6 @@ impl DerefMut for VmAllocation { } } -pub(crate) fn task_context() -> bool { - CurrentIrqArch::is_irq_enabled() - && !in_interrupt() - && ProcessManager::initialized() - && ProcessManager::current_pcb().preempt_count() == 0 -} - #[derive(Clone, Copy)] struct Plan { pages: usize, @@ -564,7 +558,7 @@ fn allocate_record( |layout| KERNEL_ALLOCATOR.alloc_metadata(layout), ) } - .map_err(|_| SystemError::ENOMEM)?; + .map_err(|_| MemoryAllocationError::Physical)?; let reservation = { POOL.lock_irqsave().reserve( plan.pages, @@ -615,9 +609,15 @@ fn allocate_record( #[cfg(feature = "mm_selftest")] test.step()?; #[cfg(feature = "mm_selftest")] - let frame = data_frame()?; + let frame = data_frame().map_err(|error| { + if error == SystemError::ENOMEM { + MemoryAllocationError::Physical + } else { + error.into() + } + })?; #[cfg(not(feature = "mm_selftest"))] - let frame = frame_zeroed()?; + let frame = frame_zeroed().map_err(|_| MemoryAllocationError::Physical)?; chunk.frames[index] = frame; chunk.acquired += 1; DATA_PAGES.fetch_add(1, Ordering::Relaxed); @@ -679,7 +679,7 @@ fn map_page( )?; #[cfg(feature = "mm_selftest")] test.step()?; - let frame = frame_zeroed()?; + let frame = frame_zeroed().map_err(|_| MemoryAllocationError::Physical)?; let table = LeafTable { record: None, frame, @@ -692,7 +692,7 @@ fn map_page( KERNEL_ALLOCATOR.alloc_metadata(layout) }) } - .map_err(|_| SystemError::ENOMEM)?, + .map_err(|_| MemoryAllocationError::Physical)?, ); let result; { diff --git a/kernel/src/mm/vmalloc/selftest.rs b/kernel/src/mm/vmalloc/selftest.rs index d29206634..5a722431f 100644 --- a/kernel/src/mm/vmalloc/selftest.rs +++ b/kernel/src/mm/vmalloc/selftest.rs @@ -776,7 +776,7 @@ fn global_header_footprint() -> bool { // Private records contain no mapped resources. This exercises the actual // intrusive queue, including producers arriving between worker pops. fn retirement_fifo() -> bool { - fn record(id: usize) -> Result, SystemError> { + fn record(id: usize) -> Result, MemoryAllocationError> { unsafe { metadata_box(AllocationRecord { start: 0, diff --git a/kernel/src/net/socket/base/scratch_selftest.rs b/kernel/src/net/socket/base/scratch_selftest.rs index 13dc9f7c9..f40aad79f 100644 --- a/kernel/src/net/socket/base/scratch_selftest.rs +++ b/kernel/src/net/socket/base/scratch_selftest.rs @@ -2,8 +2,7 @@ use super::*; use crate::{ cgroup::memory::{object::ObjectAccount, MemoryAccount}, - filesystem::vfs::file::File, - mm::kernel_buffer::KernelBuffer, + mm::{allocation::MemoryAllocationError, kernel_buffer::KernelBuffer}, net::socket::unix::datagram::UnixDatagramSocket, syscall::user_buffer::UserBuffer, }; @@ -19,14 +18,22 @@ pub(crate) fn run() -> bool { // No copy may occur: the real backing allocation below fails first. let mut output = UserBuffer::new_protected(0x1000 as *mut u8, 16384, true)?; let mut backing_attempts = 0; - let result = read_to_user_buffer_with(receiver.as_ref(), &mut output, 16384, |size| { - File::recover_io(|| { - KernelBuffer::try_new_zeroed_in_with(size, owner, &mut || { + let result = read_to_user_buffer_with( + receiver.as_ref(), + &mut output, + 16384, + |size| -> Result { + // Check one refused birth and consumer state here; bounded + // global recovery policy has separate regression coverage. + match KernelBuffer::try_new_zeroed_in_with(size, owner, &mut || { backing_attempts += 1; Err(SystemError::ENOMEM) - }) - }) - }); + }) { + Err(error @ MemoryAllocationError::Physical) => Err(error.into_errno()), + _ => Err(SystemError::EINVAL), + } + }, + ); if !matches!(result, Err(SystemError::ENOMEM)) || backing_attempts != 1 { return Ok(false); } diff --git a/kernel/src/net/socket/inet/datagram/mod.rs b/kernel/src/net/socket/inet/datagram/mod.rs index 7b638de5e..b9b577e5e 100644 --- a/kernel/src/net/socket/inet/datagram/mod.rs +++ b/kernel/src/net/socket/inet/datagram/mod.rs @@ -202,7 +202,7 @@ pub struct UdpSocketData { impl UdpSocket { pub fn new(nonblock: bool, version: IpVersion) -> Result, SystemError> { let netns = ProcessManager::current_netns(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let task = ProcessManager::current_pcb(); let account = crate::cgroup::memory::account_for_task(&task); @@ -445,7 +445,7 @@ impl UdpSocket { &self, iface: Arc, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match super::common::registration::PreparedIfaceSocket::try_new( @@ -747,7 +747,7 @@ impl UdpSocket { if limit > 1 << 30 { return Err(SystemError::ENOBUFS); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let operation = self.resize_lock.lock(); // IPv4 RX belongs to receive_queue, not the TX-only smoltcp entry. diff --git a/kernel/src/net/socket/inet/raw/socket.rs b/kernel/src/net/socket/inet/raw/socket.rs index 91cccb569..0a2cdb970 100644 --- a/kernel/src/net/socket/inet/raw/socket.rs +++ b/kernel/src/net/socket/inet/raw/socket.rs @@ -69,7 +69,7 @@ impl RawSocket { // 但为了让未 bind 的 raw socket 能接收数据包、且 poll/epoll 能正确唤醒, // 在存在 iface 时优先以“通配接收”方式附着到 loopback/默认 iface。 // 若当前 netns 尚无可用 iface,则退化为 Unbound,允许后续 bind/connect/sendto 再完成选址与附着。 - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let (memory, body, unbound, sock, me, send_account) = loop { let task = ProcessManager::current_pcb(); @@ -193,7 +193,7 @@ impl RawSocket { &self, iface: Arc, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match super::super::common::registration::PreparedIfaceSocket::try_new( diff --git a/kernel/src/net/socket/inet/stream/lifecycle.rs b/kernel/src/net/socket/inet/stream/lifecycle.rs index 71eb15f05..182204112 100644 --- a/kernel/src/net/socket/inet/stream/lifecycle.rs +++ b/kernel/src/net/socket/inet/stream/lifecycle.rs @@ -346,7 +346,7 @@ impl TcpSocket { } pub fn do_listen(&self, backlog: usize) -> Result<(), SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let unbound = matches!( self.inner.read().as_ref(), @@ -476,7 +476,7 @@ impl TcpSocket { pub fn try_accept(&self) -> Result<(Arc, smoltcp::wire::IpEndpoint), SystemError> { // Poll outside `inner`: protocol notification may acquire it again. self.netns().tcp_stack().poll(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let (handle, identity) = { let reader = self.inner.read(); diff --git a/kernel/src/net/socket/inet/stream/stream_core.rs b/kernel/src/net/socket/inet/stream/stream_core.rs index f73c5db57..1b4ab4002 100644 --- a/kernel/src/net/socket/inet/stream/stream_core.rs +++ b/kernel/src/net/socket/inet/stream/stream_core.rs @@ -255,7 +255,7 @@ impl TcpSocket { pub(super) fn prepare_bound_slot( &self, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match self.netns.tcp_stack().reserve_bound_slot(&creator) { @@ -361,7 +361,7 @@ impl TcpSocket { pub fn new(nonblock: bool, ver: smoltcp::wire::IpVersion) -> Result, SystemError> { let netns = ProcessManager::current_netns(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let task = ProcessManager::current_pcb(); let account = crate::cgroup::memory::account_for_task(&task); @@ -471,7 +471,7 @@ impl TcpSocket { Some(inner::Inner::Init(inner::Init::Unbound(_))) ) }; - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); while needs_handle() { let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); match self.netns.tcp_stack().prepare_socket_capacity(&creator) { @@ -710,7 +710,7 @@ impl TcpSocket { } fn update_socket_buffer(&self, size: usize, send: bool) -> Result<(), SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { // Domain lookup precedes even the task-only resize mutex. let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); diff --git a/kernel/src/net/socket/unix/datagram/mod.rs b/kernel/src/net/socket/unix/datagram/mod.rs index 8e6af4a62..fe1d78002 100644 --- a/kernel/src/net/socket/unix/datagram/mod.rs +++ b/kernel/src/net/socket/unix/datagram/mod.rs @@ -688,7 +688,7 @@ impl Socket for UnixDatagramSocket { fn bind(&self, endpoint: Endpoint) -> Result<(), SystemError> { let unix_endpoint = UnixEndpoint::try_from(endpoint)?; - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let (mut inner, was_unbound) = loop { let candidate = unix_endpoint.clone(); let mut inner = self.inner.lock(); diff --git a/kernel/src/net/socket/unix/mod.rs b/kernel/src/net/socket/unix/mod.rs index 5ef0873ce..b99845b9a 100644 --- a/kernel/src/net/socket/unix/mod.rs +++ b/kernel/src/net/socket/unix/mod.rs @@ -456,29 +456,13 @@ impl UnixEndpoint { path.trim(), )?; // All connect_in callers resolve before taking socket/backend - // guards. Lookup has not published the connection, so quota + // guards. Lookup has not published the connection, so allocation // recovery may retry this read-only resolution stage only. - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let inode = loop { match inode_begin.lookup_follow_symlink(&path, VFS_MAX_FOLLOW_SYMLINK_TIMES) { Ok(inode) => break inode, - Err(crate::mm::allocation::MemoryAllocationError::System(error)) => { - return Err(error); - } - Err(crate::mm::allocation::MemoryAllocationError::Charge(failure)) => { - use crate::cgroup::memory::runtime::{ - recover_charge, MemoryAllocPolicy, RecoveryDecision, - }; - match recover_charge( - &failure, - MemoryAllocPolicy::ReclaimThenOom, - 0, - &mut recovery, - )? { - RecoveryDecision::RetryOperation => continue, - RecoveryDecision::Exhausted => return Err(SystemError::ENOMEM), - } - } + Err(error) => error.recover(&mut recovery)?, } }; if inode.metadata()?.file_type != FileType::Socket { diff --git a/kernel/src/net/socket/unix/stream/mod.rs b/kernel/src/net/socket/unix/stream/mod.rs index 9a769013a..0d1350111 100644 --- a/kernel/src/net/socket/unix/stream/mod.rs +++ b/kernel/src/net/socket/unix/stream/mod.rs @@ -747,7 +747,7 @@ impl Socket for UnixStreamSocket { fn bind(&self, endpoint: Endpoint) -> Result<(), SystemError> { let addr = UnixEndpoint::try_from(endpoint)?; - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let candidate = addr.clone(); let result = { diff --git a/kernel/src/net/syscall/mod.rs b/kernel/src/net/syscall/mod.rs index 246075837..d0d0f4df9 100644 --- a/kernel/src/net/syscall/mod.rs +++ b/kernel/src/net/syscall/mod.rs @@ -26,7 +26,7 @@ pub(super) fn reserve_socket_descriptors( task: &alloc::sync::Arc, cloexec: bool, ) -> Result, system_error::SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let creator = crate::cgroup::memory::object_for_task(task); match task @@ -43,7 +43,7 @@ pub(super) fn reserve_socket_descriptors( /// socket pair. Descriptor reservations are owners, not held fd-table locks. pub(super) fn prepare_socket_file( ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { match crate::filesystem::vfs::file::PreparedFile::try_new() { Ok(prepared) => return Ok(prepared), @@ -61,7 +61,7 @@ pub(super) fn finish_socket_file( flags: crate::filesystem::vfs::file::FileFlags, mut prepared: crate::filesystem::vfs::file::PreparedFile, ) -> Result { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let result = crate::filesystem::vfs::file::File::new_socket_prepared(inode.clone(), flags, prepared); diff --git a/kernel/src/net/syscall/sys_socketpair.rs b/kernel/src/net/syscall/sys_socketpair.rs index cfbbaabd6..2823b4f5a 100644 --- a/kernel/src/net/syscall/sys_socketpair.rs +++ b/kernel/src/net/syscall/sys_socketpair.rs @@ -154,7 +154,7 @@ pub(super) fn do_socketpair( ) { return Err(SystemError::ESOCKTNOSUPPORT); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let (prepared_a, prepared_b) = loop { let a = match crate::filesystem::vfs::file::PreparedFile::try_new() { Ok(storage) => storage, diff --git a/kernel/src/process/exec.rs b/kernel/src/process/exec.rs index 2900bc2d3..d2312025c 100644 --- a/kernel/src/process/exec.rs +++ b/kernel/src/process/exec.rs @@ -399,7 +399,7 @@ impl ExecParam { if me.sighand().is_shared() { let old_sighand = me.sighand(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let new_sighand = loop { let creator = crate::cgroup::memory::object_for_task(&me); match SigHand::try_new_for(&creator) { @@ -418,7 +418,7 @@ impl ExecParam { .fd_table_snapshot() .expect("exec task has no fd table"); if shared { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let private = loop { let creator = crate::cgroup::memory::object_for_task(&me); match FileDescriptorTable::try_clone(&fd_table, None, &creator) { diff --git a/kernel/src/process/fork.rs b/kernel/src/process/fork.rs index c0ef768e6..b9990b9fb 100644 --- a/kernel/src/process/fork.rs +++ b/kernel/src/process/fork.rs @@ -385,7 +385,7 @@ impl ProcessManager { .flags .contains(CloneFlags::CLONE_PIDFD) .then(crate::mm::ucontext::AddressSpace::current); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { // Each failed unpublished birth releases both stacks and its // body before recovery. Never enter copy_process locks to reclaim. diff --git a/kernel/src/process/kstack.rs b/kernel/src/process/kstack.rs index 3255fefcb..999f738af 100644 --- a/kernel/src/process/kstack.rs +++ b/kernel/src/process/kstack.rs @@ -190,6 +190,7 @@ impl KernelStack { pub fn new() -> Result { Self::new_with_account(None).map_err(|error| match error { crate::mm::allocation::MemoryAllocationError::System(error) => error, + crate::mm::allocation::MemoryAllocationError::Physical => SystemError::ENOMEM, crate::mm::allocation::MemoryAllocationError::Charge(_) => { unreachable!("unaccounted stack cannot fail admission") } @@ -216,7 +217,14 @@ impl KernelStack { .transpose()?; if cfg!(feature = "kstack_protect") { unsafe { - let (kstack_vaddr, kstack_paddr, count) = alloc_from_kernel_space()?; + let (kstack_vaddr, kstack_paddr, count) = + alloc_from_kernel_space().map_err(|error| { + if error == SystemError::ENOMEM { + crate::mm::allocation::MemoryAllocationError::Physical + } else { + error.into() + } + })?; let real_kstack_vaddr = kstack_vaddr + KernelStack::SIZE; Ok(Self { stack: Some( @@ -234,7 +242,7 @@ impl KernelStack { stack: None, dynamic_stack: Some(unsafe { PhysicalBox::::try_new_zeroed() - .map_err(|_| SystemError::ENOMEM)? + .map_err(|_| crate::mm::allocation::MemoryAllocationError::Physical)? .assume_init() }), ty: KernelStackType::Dynamic, diff --git a/kernel/src/process/namespace/setns.rs b/kernel/src/process/namespace/setns.rs index ee03b699f..0d3d0be67 100644 --- a/kernel/src/process/namespace/setns.rs +++ b/kernel/src/process/namespace/setns.rs @@ -68,7 +68,7 @@ fn install_setns( new_cred: Option>, flags: CloneFlags, ) -> Result<(), SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let mut new_fs = None; let fs_refs = lock_fs_refs_copy(); diff --git a/kernel/src/process/namespace/unshare.rs b/kernel/src/process/namespace/unshare.rs index 1616a938a..a92c8d22c 100644 --- a/kernel/src/process/namespace/unshare.rs +++ b/kernel/src/process/namespace/unshare.rs @@ -21,7 +21,7 @@ pub fn ksys_unshare(flags: CloneFlags) -> Result<(), SystemError> { check_unshare_flags(flags)?; let current_pcb = ProcessManager::current_pcb(); - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); loop { let new_cred = unshare_user_cred(flags, ¤t_pcb)?.map(Cred::new_arc); // Retain completed candidates outside the topology barrier, even if diff --git a/kernel/src/process/pid_selftest.rs b/kernel/src/process/pid_selftest.rs index cb8bf27c6..4ea4fab4b 100644 --- a/kernel/src/process/pid_selftest.rs +++ b/kernel/src/process/pid_selftest.rs @@ -5,7 +5,7 @@ use super::*; use crate::process::KernelStack; pub(crate) fn task() -> Result, SystemError> { - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let task = loop { let birth = (|| { let resources = crate::process::task::PreparedTaskResources::try_for_fork( diff --git a/kernel/src/process/posix_timer/cpu.rs b/kernel/src/process/posix_timer/cpu.rs index 5e76e1a4a..c8699512b 100644 --- a/kernel/src/process/posix_timer/cpu.rs +++ b/kernel/src/process/posix_timer/cpu.rs @@ -28,7 +28,7 @@ impl CpuBackend { } else { task.process_signal().cpu_event_domain() }; - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::default(); let registration = loop { match CpuEventDomain::prepare_registration(&object_for_task(pcb), callback.clone()) { Ok(registration) => break registration, diff --git a/kernel/src/process/syscall/sys_pidfdopen.rs b/kernel/src/process/syscall/sys_pidfdopen.rs index cb3d0621d..be355d4db 100644 --- a/kernel/src/process/syscall/sys_pidfdopen.rs +++ b/kernel/src/process/syscall/sys_pidfdopen.rs @@ -54,7 +54,7 @@ impl Syscall for SysPidFdOpen { if pid.pid_task(PidType::TGID).is_none() { return Err(SystemError::EINVAL); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let prepared = loop { match PidFd::prepare(¤t, pid.clone(), file_flags, false) { Ok(prepared) => break prepared, diff --git a/kernel/src/process/usermodehelper.rs b/kernel/src/process/usermodehelper.rs index ddb5282d4..80cb443bc 100644 --- a/kernel/src/process/usermodehelper.rs +++ b/kernel/src/process/usermodehelper.rs @@ -214,7 +214,7 @@ fn run_child_prepare( if let Some(cred) = cred { child.install_cred(cred); } - let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut recovery = crate::mm::allocation::AllocationRecoveryState::new(); let fd_table = loop { // No PCB basic-info, fd-table, or helper-result guard spans admission // or recovery. Preserve the old table until the private one is ready. diff --git a/kernel/src/syscall/user_access.rs b/kernel/src/syscall/user_access.rs index 729c48f72..b4020afba 100644 --- a/kernel/src/syscall/user_access.rs +++ b/kernel/src/syscall/user_access.rs @@ -102,7 +102,7 @@ fn prefault_user_range_mode( }; let mut current = start_page; let mut retried = false; - let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new(); + let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new(); 'retry: loop { let creator = crate::cgroup::memory::object_for_task( &crate::process::ProcessManager::current_pcb(), @@ -186,6 +186,19 @@ fn prefault_user_range_mode( )?; continue 'retry; } + Err( + error @ crate::mm::allocation::MemoryAllocationError::Physical, + ) => { + if let Err(error) = error.recover(&mut charge_recovery) { + crate::mm::fault::account_fault_recovery_failure( + &data_account, + fault_flags, + error.clone(), + ); + return Err(error); + } + continue 'retry; + } Err(crate::mm::allocation::MemoryAllocationError::System( error, )) => Err(error), diff --git a/kernel/src/time/cpu_events/mod.rs b/kernel/src/time/cpu_events/mod.rs index a8f20862c..a3b6efa9e 100644 --- a/kernel/src/time/cpu_events/mod.rs +++ b/kernel/src/time/cpu_events/mod.rs @@ -3,6 +3,7 @@ //! Producers only publish counters and schedule one preborn tasklet. The lock //! order is domain queues -> registration state. Callbacks and final drops are //! outside both guards; registrations do not retain their domain. +use crate::mm::allocation::AllocationRecoveryState; mod queue; use alloc::sync::Arc; @@ -10,10 +11,7 @@ use core::sync::atomic::{fence, AtomicU64, Ordering}; use system_error::SystemError; use crate::{ - cgroup::memory::{ - accounting::KernelByteKind, object::ObjectAccount, owned::OwnedBox, - runtime::ChargeRecoveryState, - }, + cgroup::memory::{accounting::KernelByteKind, object::ObjectAccount, owned::OwnedBox}, exception::tasklet::{tasklet_schedule, Tasklet}, libs::{ spinlock::SpinLock, @@ -349,7 +347,7 @@ impl CpuEventDomain { return Ok(()); } let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb()); - let mut recovery = ChargeRecoveryState::new(); + let mut recovery = AllocationRecoveryState::new(); loop { match self.wait_once(kind, deadline, &mut condition, &creator) { Ok(()) => return Ok(()), diff --git a/user/apps/tests/dunitest/suites/normal/global_reclaim.cc b/user/apps/tests/dunitest/suites/normal/global_reclaim.cc index 046920b98..fbb01fb2c 100644 --- a/user/apps/tests/dunitest/suites/normal/global_reclaim.cc +++ b/user/apps/tests/dunitest/suites/normal/global_reclaim.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -9,9 +10,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -30,10 +33,93 @@ uint64_t MemTotalKiB() { // Keep this capacity workload unchanged between baseline and fixed kernels. // Dirty mode additionally exercises writeback progress before physical retry. -int ExerciseReclaim(int fd, bool dirty) { - constexpr size_t file_bytes = 192 * kMiB; +int ExerciseReclaim(int fd, bool dirty, bool syscall_io) { + const size_t file_bytes = (syscall_io ? 128 : 192) * kMiB; constexpr size_t anonymous_bytes = 700 * kMiB; if (ftruncate(fd, file_bytes) != 0) return 10; + if (syscall_io) { + // Materialize extents before pressure so this regression measures cache + // admission rather than per-block sparse-growth metadata transactions. + // The pressure phase must still allocate cache pages for these extents. + { + unsigned char initial[64 * 1024] = {}; + for (size_t offset = 0; offset < file_bytes; offset += sizeof(initial)) { + if (pwrite(fd, initial, sizeof(initial), offset) != sizeof(initial)) + return 29; + } + if (fsync(fd) != 0) return 30; + if (posix_fadvise(fd, 0, file_bytes, POSIX_FADV_DONTNEED) != 0) return 31; + } + fprintf(stderr, "file reclaim preparation complete\n"); + // Keep anonymous memory resident while the backing cache competes for + // frames. Unlike the mmap tests below, every new file page is allocated + // by pread/pwrite, outside the user page-fault recovery path. + void* anonymous = mmap(nullptr, anonymous_bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anonymous == MAP_FAILED) return 20; + auto* resident = static_cast(anonymous); + for (size_t offset = 0; offset < anonymous_bytes; offset += 4096) + resident[offset] = 37; + unsigned char buffer[4096]; + for (size_t offset = 0; offset < file_bytes; offset += sizeof(buffer)) { + const unsigned char pattern = + static_cast(offset / sizeof(buffer)); + memset(buffer, pattern, sizeof(buffer)); + const ssize_t written = pwrite(fd, buffer, sizeof(buffer), offset); + if (written != sizeof(buffer)) { + const int error = written < 0 ? errno : 0; + fprintf(stderr, "pwrite offset=%zu result=%zd errno=%d\n", + offset, written, error); + return 21; + } + } + fprintf(stderr, "file reclaim write complete\n"); + if (fsync(fd) != 0) { + fprintf(stderr, "fsync errno=%d\n", errno); + return 22; + } + fprintf(stderr, "file reclaim fsync complete\n"); + // Exercise short thread lifetimes while anonymous memory remains + // resident and the file cache has just been populated. This covers + // usability under pressure, not deterministic stack fragmentation. + for (unsigned int attempt = 0; attempt < 4; ++attempt) { + pthread_t thread; + int error = pthread_create( + &thread, nullptr, [](void* value) -> void* { return value; }, buffer); + if (error != 0) { + fprintf(stderr, "pthread_create attempt=%u error=%d\n", attempt, error); + return 27; + } + void* returned = nullptr; + error = pthread_join(thread, &returned); + if (error != 0 || returned != buffer) { + fprintf(stderr, "pthread_join attempt=%u error=%d value_matches=%d\n", + attempt, error, returned == buffer); + return 28; + } + } + fprintf(stderr, "file reclaim threads complete\n"); + for (size_t end = file_bytes; end != 0; end -= sizeof(buffer)) { + const size_t offset = end - sizeof(buffer); + const unsigned char pattern = + static_cast(offset / sizeof(buffer)); + const ssize_t read = pread(fd, buffer, sizeof(buffer), offset); + if (read != sizeof(buffer)) { + const int error = read < 0 ? errno : 0; + fprintf(stderr, "pread offset=%zu result=%zd errno=%d\n", + offset, read, error); + return 23; + } + for (unsigned char value : buffer) { + if (value != pattern) return 24; + } + } + fprintf(stderr, "file reclaim read complete\n"); + for (size_t offset = 0; offset < anonymous_bytes; offset += 4096) { + if (resident[offset] != 37) return 25; + } + return munmap(anonymous, anonymous_bytes) == 0 ? 0 : 26; + } void* file = mmap(nullptr, file_bytes, PROT_READ | (dirty ? PROT_WRITE : 0), MAP_SHARED, fd, 0); if (file == MAP_FAILED) return 11; @@ -83,7 +169,7 @@ bool WriteText(const std::string& path, const std::string& value) { return written && closed; } -void CheckReclaim(bool dirty, bool protect_low = false) { +void CheckReclaim(bool dirty, bool protect_low = false, bool syscall_io = false) { // A capacity regression, run in the dedicated 1 GiB guest. Larger guests // would pass without entering physical reclaim and provide false coverage. const uint64_t total = MemTotalKiB(); @@ -135,7 +221,7 @@ void CheckReclaim(bool dirty, bool protect_low = false) { if (child == 0) { if (protect_low && !WriteText(group + "/cgroup.procs", std::to_string(getpid()))) _exit(19); - _exit(ExerciseReclaim(fd, dirty)); + _exit(ExerciseReclaim(fd, dirty, syscall_io)); } if (child < 0) { close(fd); @@ -146,7 +232,12 @@ void CheckReclaim(bool dirty, bool protect_low = false) { } int status = 0; pid_t result = 0; - for (int i = 0; i < 12000 && result == 0; ++i) { + // The syscall case includes extent preparation and two full syncs. This is + // its whole-test budget, independent of the kernel's physical retry bound. + const int timeout_seconds = syscall_io ? 300 : 120; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(timeout_seconds); + while (result == 0 && std::chrono::steady_clock::now() < deadline) { result = waitpid(child, &status, WNOHANG); if (result < 0 && errno == EINTR) result = 0; if (result == 0) usleep(10000); @@ -167,10 +258,12 @@ void CheckReclaim(bool dirty, bool protect_low = false) { EXPECT_GT(low, 0u) << "must exercise low-protection fallback"; events.close(); EXPECT_EQ(rmdir(group.c_str()), 0) << errno; - if (enabled_memory) + if (enabled_memory) { EXPECT_TRUE(WriteText("/sys/fs/cgroup/cgroup.subtree_control", "-memory")); + } } - ASSERT_EQ(result, child) << "reclaim child failed to finish within 120s"; + ASSERT_EQ(result, child) << "reclaim child failed to finish within " + << timeout_seconds << "s"; ASSERT_TRUE(WIFEXITED(status)) << "unexpected signal: " << status; EXPECT_EQ(WEXITSTATUS(status), 0); EXPECT_EQ(OomKills(), before) << "reclaimable cache must not cause an OOM kill"; @@ -183,6 +276,12 @@ TEST(GlobalReclaim, CleanCacheRemainsReclaimableWhenPreparationCannotAllocate) { TEST(GlobalReclaim, DirtyCacheMakesProgressBeforeGlobalOom) { CheckReclaim(true); } +TEST(GlobalReclaim, FileSyscallAllocationsRecoverUnderGlobalPressure) { + // 128 MiB crosses the reproduced file-allocation failure point while + // keeping this syscall regression bounded. The separate 192 MiB mmap + // cases above exercise the larger reclaim inventory. + CheckReclaim(false, false, true); +} TEST(GlobalReclaim, GlobalPressureCanBreachLowAcrossBoundedScanBatches) { // 192 MiB is 49,152 pages, well beyond one 4,096-candidate scan. // memory.low must yield under global pressure; memory.min remains hard.