fix(mm): recover physical allocations at lock-free transaction boundaries

kernel/src/arch/riscv64/mm/fault.rs

@@ -1,6 +1,6 @@
+use crate::mm::allocation::AllocationRecoveryState;
 use crate::{
     arch::ipc::signal::Signal,
-    cgroup::memory::runtime::ChargeRecoveryState,
     ipc::signal_types::{BUS_ADRERR, SEGV_ACCERR, SEGV_MAPERR},
     mm::{
         allocation::MemoryAllocationError,
@@ -72,7 +72,7 @@ impl RiscV64MMArch {
             }
             _ => unreachable!("not a user page fault"),
         };
-        let mut charge_recovery = ChargeRecoveryState::new();
+        let mut charge_recovery = AllocationRecoveryState::new();
         let mut global_recovery = GlobalFaultRecoveryState::new();
         loop {
             let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb());
@@ -122,7 +122,8 @@ impl RiscV64MMArch {
                         Err(MemoryAllocationError::System(
                             SystemError::EINTR | SystemError::ERESTARTSYS,
                         )) => return Ok(()),
-                        Err(MemoryAllocationError::System(SystemError::ENOMEM)) => {
+                        Err(MemoryAllocationError::System(SystemError::ENOMEM))
+                        | Err(MemoryAllocationError::Physical) => {
                             reason = VmFaultReason::VM_FAULT_OOM
                         }
                         Err(MemoryAllocationError::System(_)) => {
@@ -179,7 +180,7 @@ impl RiscV64MMArch {
 /// recovery yields to pending signals and a later hardware fault, as on x86.
 fn recover_charge(
     failure: crate::cgroup::memory::accounting::ChargeFailure,
-    state: &mut ChargeRecoveryState,
+    state: &mut AllocationRecoveryState,
     account: &crate::cgroup::memory::MemoryAccount,
     flags: FaultFlags,
 ) -> bool {

kernel/src/arch/x86_64/mm/fault.rs

@@ -419,7 +419,7 @@ impl X86_64MMArch {
         };
 
         let mut global_recovery = crate::mm::fault_recovery::GlobalFaultRecoveryState::new();
-        let mut charge_recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new();
+        let mut charge_recovery = crate::mm::allocation::AllocationRecoveryState::new();
 
         'fault_retry: loop {
             let creator = crate::cgroup::memory::object_for_task(&ProcessManager::current_pcb());
@@ -717,7 +717,8 @@ impl X86_64MMArch {
                             }
                             Err(crate::mm::allocation::MemoryAllocationError::System(
                                 SystemError::ENOMEM,
-                            )) => {
+                            ))
+                            | Err(crate::mm::allocation::MemoryAllocationError::Physical) => {
                                 fault = VmFaultReason::VM_FAULT_OOM;
                                 crate::mm::fault::account_fault_result(&data_account, flags, fault);
                                 space_guard = current_address_space.write();

kernel/src/driver/base/block/gendisk/cache.rs

@@ -52,10 +52,9 @@ where
         };
         let confirmed_mask = (((1u16 << confirmed_last) - 1) ^ ((1u16 << first) - 1)) as u8;
         *remaining &= !confirmed_mask;
-        result.sync_result.map_err(|error| match error {
-            MemoryAllocationError::System(error) => error,
-            MemoryAllocationError::Charge(_) => SystemError::ENOMEM,
-        })?;
+        result
+            .sync_result
+            .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?;
         if result.written_len != end - start {
             return Err(SystemError::EIO);
         }
@@ -149,10 +148,9 @@ impl PageCacheBackend for RawBlockBackend {
         // A completed result belongs to the lower owner's exact-sync protocol
         // (loop's write_piece already performs that recovery); never replay it.
         let result = File::recover_io(|| disk.write_payload(offset, bytes, WriteSyncIntent::None))?;
-        result.sync_result.map_err(|error| match error {
-            MemoryAllocationError::System(error) => error,
-            MemoryAllocationError::Charge(_) => SystemError::ENOMEM,
-        })?;
+        result
+            .sync_result
+            .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?;
         Ok(result.written_len)
     }
 

kernel/src/driver/base/block/gendisk/mod.rs

@@ -280,10 +280,9 @@ impl GenDisk {
         })?;
         // The lower owner has already submitted this prefix and owns exact
         // sync recovery. Its completion error must never replay the payload.
-        result.sync_result.map_err(|error| match error {
-            crate::mm::allocation::MemoryAllocationError::System(error) => error,
-            crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM,
-        })?;
+        result
+            .sync_result
+            .map_err(crate::mm::allocation::MemoryAllocationError::into_errno)?;
         Ok(result.written_len)
     }
 

kernel/src/driver/block/loop_device/io.rs

@@ -168,8 +168,19 @@ impl LoopDevice {
         })
         .map_err(Self::backing_error)?;
         result.sync_result = Self::normalize_backing_sync_result(result.sync_result);
-        let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default();
+        let mut recovery = crate::mm::allocation::AllocationRecoveryState::default();
         while let Err(error) = core::mem::replace(&mut result.sync_result, Ok(())) {
+            if matches!(error, MemoryAllocationError::Physical) {
+                // The loop pin/gate and NOIO scope still belong to our caller.
+                // Let its lock-free boundary finish the retained sync range;
+                // never consume the deferred completions or replay payload.
+                result.sync_result = Err(if result.sync_range.is_some() {
+                    error
+                } else {
+                    error.into_errno().into()
+                });
+                break;
+            }
             // Payload is committed: recover only its retained exact sync range.
             if let Err(error) = error.recover(&mut recovery) {
                 result.sync_result = Err(error.into());
@@ -254,7 +265,12 @@ impl LoopDevice {
                 Ok(result) => result,
                 // This is a data error, not a recoverable sync failure. No
                 // invented sync_range may replay an unsubmitted remainder.
-                Err(error) if done != 0 => return Ok(DelegatedWriteResult::new(done, Err(error))),
+                Err(error) if done != 0 => {
+                    return Ok(DelegatedWriteResult::new(
+                        done,
+                        Err(error.into_errno().into()),
+                    ))
+                }
                 Err(error) => return Err(error),
             };
             let expected = if full { count } else { LBA_SIZE };
@@ -317,23 +333,45 @@ impl LoopDevice {
     fn recover_backing_io<T>(
         mut attempt: impl FnMut() -> Result<T, MemoryAllocationError>,
     ) -> Result<T, MemoryAllocationError> {
-        let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::default();
+        let mut recovery = crate::mm::allocation::AllocationRecoveryState::default();
         loop {
             match attempt() {
                 Ok(value) => return Ok(value),
+                // Every caller retains loop admission and a NOIO scope. A
+                // physical failure must escape with its provenance intact so
+                // the outer transaction can reclaim after releasing them.
+                Err(error @ MemoryAllocationError::Physical) => return Err(error),
                 Err(error) => error.recover(&mut recovery)?,
             }
         }
     }
 
+    #[cfg(feature = "mm_selftest")]
+    pub(crate) fn physical_boundary_selftest() -> bool {
+        let _noio = crate::cgroup::memory::io::MemoryIoScope::enter_noio();
+        let mut attempts = 0;
+        let result: Result<(), MemoryAllocationError> = Self::recover_backing_io(|| {
+            attempts += 1;
+            Err(MemoryAllocationError::Physical)
+        });
+        attempts == 1
+            && matches!(result, Err(MemoryAllocationError::Physical))
+            && matches!(
+                Self::normalize_backing_sync_result(Err(MemoryAllocationError::Physical)),
+                Err(MemoryAllocationError::Physical)
+            )
+    }
+
     fn normalize_backing_sync_result(
         result: Result<(), MemoryAllocationError>,
     ) -> Result<(), MemoryAllocationError> {
         // Linux keeps -EINVAL in lo_req_flush, but errno_to_blk_status maps
         // it to BLK_STS_IOERR. No backing flush error is a successful flush.
         match result {
             Ok(()) => Ok(()),
-            Err(error @ MemoryAllocationError::Charge(_)) => Err(error),
+            Err(error @ (MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical)) => {
+                Err(error)
+            }
             Err(MemoryAllocationError::System(_)) => Err(SystemError::EIO.into()),
         }
     }
@@ -361,14 +399,16 @@ impl LoopDevice {
                 | SystemError::ETIME => error.into(),
                 _ => SystemError::EIO.into(),
             },
-            error @ MemoryAllocationError::Charge(_) => error,
+            error @ (MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => error,
         }
     }
 
     pub(super) fn legacy_io_terminal(error: MemoryAllocationError) -> SystemError {
         match error {
             MemoryAllocationError::System(error) => error,
-            MemoryAllocationError::Charge(_) => SystemError::ENOMEM,
+            MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical => {
+                SystemError::ENOMEM
+            }
         }
     }
 }

kernel/src/filesystem/epoll/event_poll.rs

@@ -297,7 +297,7 @@ impl EventPoll {
             if !dst_file.supports_poll() {
                 return Err(SystemError::ENOSYS);
             }
-            let mut recovery = crate::cgroup::memory::runtime::ChargeRecoveryState::new();
+            let mut recovery = crate::mm::allocation::AllocationRecoveryState::new();
             loop {
                 let (ready_state, poll_epitems) = {
                     let guard = if nonblock {

kernel/src/filesystem/overlayfs/file.rs

@@ -128,10 +128,7 @@ pub(super) fn write_at(
     // The legacy scalar inode ABI cannot retain a completed write while
     // unwinding for recovery. A backing sync charge is a terminal failure;
     // returning it as recoverable would repeat the already committed data.
-    result.sync_result.map_err(|error| match error {
-        crate::mm::allocation::MemoryAllocationError::System(error) => error,
-        crate::mm::allocation::MemoryAllocationError::Charge(_) => SystemError::ENOMEM,
-    })?;
+    result.sync_result.map_err(|error| error.into_errno())?;
     Ok(result.written_len)
 }
 

kernel/src/filesystem/overlayfs/metadata.rs

@@ -349,10 +349,7 @@ impl XattrBuffer {
 /// backing errno, and report rejected scratch admission without reclaim/OOM
 /// under the caller's resource locks or overridden backing credentials.
 pub(super) fn xattr_read_error(error: MemoryAllocationError) -> SystemError {
-    match error {
-        MemoryAllocationError::System(error) => error,
-        MemoryAllocationError::Charge(_) => SystemError::ENOMEM,
-    }
+    error.into_errno()
 }
 
 fn for_each_xattr_name<E: From<SystemError>>(

kernel/src/filesystem/page_cache.rs

@@ -2081,7 +2081,7 @@ impl PageEntry {
                 accounted_unevictable: AtomicBool::new(false),
                 active_users: AtomicUsize::new(0),
             })
-            .map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM))?;
+            .map_err(|_| MemoryAllocationError::Physical)?;
         Ok(Self { storage })
     }
 
@@ -2635,6 +2635,8 @@ impl PageCache {
         let mut pages = AccountedPageMap::new_in(HeapPlanAllocator::Auto);
         pages
             .try_reserve(required)
+            // Auto backing may fail its virtual-address reservation. Preserve
+            // that opaque allocation failure rather than guessing physical.
             .map_err(|_| SystemError::ENOMEM)?;
         let (pointer, layout) = pages.raw_table().allocation_info();
         let charge = match unsafe {
@@ -2658,9 +2660,12 @@ impl PageCache {
                     HeapPlan::Physical.footprint(layout),
                 )?;
                 pages = AccountedPageMap::new_in(HeapPlanAllocator::Physical);
-                pages
-                    .try_reserve(required)
-                    .map_err(|_| SystemError::ENOMEM)?;
+                pages.try_reserve(required).map_err(|error| match error {
+                    hashbrown::TryReserveError::CapacityOverflow => SystemError::ENOMEM.into(),
+                    hashbrown::TryReserveError::AllocError { .. } => {
+                        MemoryAllocationError::Physical
+                    }
+                })?;
                 let (direct_pointer, actual_layout) = pages.raw_table().allocation_info();
                 assert_eq!(
                     actual_layout, layout,
@@ -3196,10 +3201,7 @@ impl PageCache {
     }
 
     fn legacy_allocation_error(error: MemoryAllocationError) -> SystemError {
-        match error {
-            MemoryAllocationError::Charge(_) => SystemError::ENOMEM,
-            MemoryAllocationError::System(error) => error,
-        }
+        error.into_errno()
     }
 
     fn allocate_page_with_owner(
@@ -3227,13 +3229,12 @@ impl PageCache {
                 charge,
             )
         } else {
-            crate::mm::page::allocate_registered_data_page(
+            crate::mm::page::allocate_registered_data_page_typed(
                 page_type,
                 self.page_flags(),
                 Some(charge),
             )
         }
-        .map_err(MemoryAllocationError::System)
     }
 
     fn prepare_entry(
@@ -3242,9 +3243,8 @@ impl PageCache {
         state: PageState,
         creator: &Arc<ObjectAccount>,
     ) -> Result<Arc<PageEntry>, MemoryAllocationError> {
-        let result = PageEntry::try_new(page.clone(), state, creator).and_then(|entry| {
-            Arc::try_new(entry).map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM))
-        });
+        let result = PageEntry::try_new(page.clone(), state, creator)
+            .and_then(|entry| Arc::try_new(entry).map_err(|_| MemoryAllocationError::Physical));
         if result.is_err() {
             self.discard_unlinked_page(page);
         }

kernel/src/filesystem/page_cache/ordered_index.rs

@@ -5,7 +5,6 @@ use crate::cgroup::memory::{accounting::KernelByteKind, object::ObjectAccount, o
 use crate::mm::allocation::MemoryAllocationError;
 use alloc::sync::Arc;
 use core::ops::{Bound, RangeBounds};
-use system_error::SystemError;
 
 #[derive(Debug)]
 pub(super) struct Node {
@@ -27,7 +26,7 @@ impl Node {
                 right: None,
                 height: 1,
             })
-            .map_err(|_| MemoryAllocationError::System(SystemError::ENOMEM))
+            .map_err(|_| MemoryAllocationError::Physical)
     }
 
     fn height(node: &Option<OwnedBox<Self>>) -> i32 {

kernel/src/filesystem/vfs/file.rs

@@ -1070,9 +1070,12 @@ impl File {
     ) -> Result<(), MemoryAllocationError> {
         let inode = self.inode();
         let sync_result = inode.sync_file_range(start, end, datasync, self.private_data.lock());
-        // Charge is an internal retry, not the final fsync result. Preserve
+        // Recoverable allocation failure is not the final fsync result. Preserve
         // the OFD cursor until synchronization reaches a terminal result.
-        if matches!(&sync_result, Err(MemoryAllocationError::Charge(_))) {
+        if sync_result
+            .as_ref()
+            .is_err_and(|error| error.is_recoverable())
+        {
             return sync_result;
         }
         let wb_result = match inode.page_cache() {
@@ -1313,12 +1316,10 @@ impl File {
                 }
                 Ok(false) => {}
                 Err(error) => {
-                    // Data is already committed. Never let a metadata Charge
-                    // escape to the primary-write recovery loop and replay it.
-                    result.sync_result = Err(match error {
-                        MemoryAllocationError::Charge(_) => SystemError::ENOMEM.into(),
-                        error => error,
-                    });
+                    // Data is already committed and there is no continuation
+                    // for this metadata operation. A sync retry cannot finish
+                    // it, and a primary-write retry would duplicate the data.
+                    result.sync_result = Err(error.into_errno().into());
                     return;
                 }
             }
@@ -2607,7 +2608,7 @@ impl File {
                 result.completion = None;
                 result.deferred.clear();
             }
-            Err(MemoryAllocationError::Charge(_)) => {
+            Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => {
                 if let Some(completion) = result.completion.take() {
                     if result.deferred.try_reserve(1).is_err() {
                         result.sync_result = Err(SystemError::ENOMEM.into());
@@ -2740,7 +2741,9 @@ impl File {
                 Ok(_) => {}
                 // Speculative prefetch never kills a task for an optional page.
                 // Required read_at admission is handled by the inode outer.
-                Err(crate::mm::allocation::MemoryAllocationError::Charge(_)) => return Ok(()),
+                Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => {
+                    return Ok(())
+                }
                 Err(crate::mm::allocation::MemoryAllocationError::System(error)) => {
                     return Err(error)
                 }
@@ -2760,7 +2763,9 @@ impl File {
                 &creator,
             ) {
                 Ok(_) => {}
-                Err(crate::mm::allocation::MemoryAllocationError::Charge(_)) => return Ok(()),
+                Err(MemoryAllocationError::Charge(_) | MemoryAllocationError::Physical) => {
+                    return Ok(())
+                }
                 Err(crate::mm::allocation::MemoryAllocationError::System(error)) => {
                     return Err(error)
                 }

kernel/src/filesystem/vfs/file/allocation.rs

@@ -17,7 +17,7 @@ impl PreparedFile {
     /// Guardless storage-only recovery. Call before instance reservations,
     /// inode open side effects and publication; this does not open a File.
     pub(crate) fn new_recovering() -> Result<Self, system_error::SystemError> {
-        let mut state = crate::cgroup::memory::runtime::ChargeRecoveryState::default();
+        let mut state = crate::mm::allocation::AllocationRecoveryState::default();
         loop {
             match Self::try_new() {
                 Ok(storage) => return Ok(storage),