fix(kernel): preserve user-return work and retire stopped TLB targets

kernel/src/debug/mod.rs

@@ -14,6 +14,8 @@ pub mod page_cache;
 pub mod panic;
 pub mod rcu;
 pub mod rng;
+#[cfg(feature = "sched_selftest")]
+pub mod sched;
 pub mod sysfs;
 pub mod task_dump;
 pub mod timekeeping;

kernel/src/debug/sched.rs

@@ -0,0 +1,75 @@
+//! Scheduler user-return probes, present only in sched_selftest builds.
+
+use alloc::string::ToString;
+use system_error::SystemError;
+
+use crate::{
+    debug::sysfs::debugfs_kobj,
+    driver::base::kobject::KObject,
+    filesystem::{
+        kernfs::callback::{KernCallbackData, KernFSCallback, KernFilePrivateData},
+        vfs::{InodeMode, PollStatus},
+    },
+};
+
+#[derive(Debug)]
+struct RseqNotifySelftest;
+
+impl KernFSCallback for RseqNotifySelftest {
+    fn open(&self, mut data: KernCallbackData) -> Result<(), SystemError> {
+        let report = crate::process::rseq::selftest::run()?;
+        data.file_private_data_mut()
+            .replace(KernFilePrivateData::DebugTextSnapshot(report));
+        Ok(())
+    }
+
+    fn read(
+        &self,
+        data: KernCallbackData,
+        buf: &mut [u8],
+        offset: usize,
+    ) -> Result<usize, SystemError> {
+        let report = match data.file_private_data() {
+            Some(KernFilePrivateData::DebugTextSnapshot(report)) => report,
+            _ => return Err(SystemError::EINVAL),
+        };
+        let bytes = report.as_bytes();
+        if offset >= bytes.len() {
+            return Ok(0);
+        }
+        let count = buf.len().min(bytes.len() - offset);
+        buf[..count].copy_from_slice(&bytes[offset..offset + count]);
+        Ok(count)
+    }
+
+    fn write(
+        &self,
+        _data: KernCallbackData,
+        _buf: &[u8],
+        _offset: usize,
+    ) -> Result<usize, SystemError> {
+        Err(SystemError::EPERM)
+    }
+
+    fn poll(&self, _data: KernCallbackData) -> Result<PollStatus, SystemError> {
+        Ok(PollStatus::READ)
+    }
+}
+
+pub fn init_debugfs_sched() -> Result<(), SystemError> {
+    let root = debugfs_kobj().inode().ok_or(SystemError::ENOENT)?;
+    let sched = root.add_dir(
+        "sched".to_string(),
+        InodeMode::from_bits_truncate(0o500),
+        None,
+        None,
+    )?;
+    sched.add_file(
+        "rseq_notify_selftest".to_string(),
+        InodeMode::S_IRUSR,
+        Some(256),
+        None,
+        Some(&RseqNotifySelftest),
+    )?;
+    Ok(())
+}

kernel/src/debug/sysfs/mod.rs

@@ -29,6 +29,8 @@ fn debugfs_init() -> Result<(), SystemError> {
     super::fuse::init_debugfs_fuse()?;
     super::kthread::init_debugfs_kthread()?;
     super::rng::init_debugfs_rng()?;
+    #[cfg(feature = "sched_selftest")]
+    super::sched::init_debugfs_sched()?;
     #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))]
     super::mm::init_debugfs_mm()?;
     super::page_cache::init_debugfs_page_cache()?;

kernel/src/exception/entry.rs

@@ -60,6 +60,21 @@ unsafe fn exit_to_user_mode_loop(frame: &mut TrapFrame, mut process_flags_work:
         if process_flags_work.contains(ProcessFlags::NEED_SCHEDULE) {
             schedule(SchedMode::SM_PREEMPT);
         } else {
+            if process_flags_work.contains(ProcessFlags::NEED_CHILD_TID) {
+                let pcb = ProcessManager::current_pcb();
+                pcb.flags().remove(ProcessFlags::NEED_CHILD_TID);
+                let addr = pcb.take_set_child_tid();
+                if let Some(addr) = addr {
+                    let child_tid = pcb.task_pid_vnr().data() as i32;
+                    // Like Linux schedule_tail(), write before delivering
+                    // signals, outside the switch locks and with faults enabled.
+                    CurrentIrqArch::interrupt_enable();
+                    let _ =
+                        crate::syscall::user_access::write_one_to_user_protected(addr, &child_tid);
+                    CurrentIrqArch::interrupt_disable();
+                }
+                process_flags_work = pcb.flags().load();
+            }
             if process_flags_work.contains(ProcessFlags::NEED_MEMORY_HIGH) {
                 CurrentIrqArch::interrupt_enable();
                 crate::cgroup::memory::high::handle_user_return();
@@ -69,7 +84,14 @@ unsafe fn exit_to_user_mode_loop(frame: &mut TrapFrame, mut process_flags_work:
             // 优先处理 rseq，因为信号递送会保存 trapframe 到 sigframe
             // rseq 的 IP fixup 必须在信号递送之前完成
             if process_flags_work.contains(ProcessFlags::NEED_RSEQ) {
+                // Consume before enabling IRQs: preemption/migration while
+                // touching user memory must leave a new notification pending.
+                ProcessManager::current_pcb()
+                    .flags()
+                    .remove(ProcessFlags::NEED_RSEQ);
+                CurrentIrqArch::interrupt_enable();
                 let _ = Rseq::handle_notify_resume(Some(frame));
+                CurrentIrqArch::interrupt_disable();
                 process_flags_work = ProcessManager::current_pcb().flags().load();
             }
 

kernel/src/mm/tlb.rs

@@ -11,7 +11,8 @@
 //! - INV-1: At any point, if CPU `c`'s hardware page table equals `mm.table_paddr`, then bit `c` in `mm.active_cpus` is 1.
 //! - INV-2: `flush_tlb_*` must publish page table writes first, then `inc_mm_tlb_gen`, then read `active_cpus`.
 //! - INV-3: Freeing physical pages / page-table pages must happen after shootdown completion (guaranteed by `MmuGather`).
-//! - INV-4: When `flush_tlb_multi` returns, all target CPUs have finished executing `local_flush_tlb_func`.
+//! - INV-4: On return, each selected target has acknowledged invalidation or entered irreversible `Dead` state;
+//!   no CSD retains the request's stack pointer.
 //! - INV-5: When `freed_tables == true`, the receiving end must handle it (simplified: in this iteration the sender
 //!   sends to all active CPUs regardless, and the receiver unconditionally executes).
 
@@ -52,6 +53,8 @@ use crate::{
 
 #[cfg(target_arch = "x86_64")]
 use crate::libs::spinlock::{SpinLock, SpinLockGuard};
+#[cfg(target_arch = "x86_64")]
+use crate::smp::cpu::CpuHpState;
 
 /// Sentinel end value meaning "flush the entire mm"; combined with start=0 it means "do not use range-based flush".
 pub const TLB_FLUSH_ALL: VirtAddr = VirtAddr::new(usize::MAX);
@@ -170,13 +173,17 @@ pub fn tlb_state_loaded_mm() -> Option<Arc<AddressSpace>> {
 /// 这在语义上仍然正确：页表写入发生在 `done=false`（Release）之前，
 /// handler 的 Acquire 读保证能看到新页表，提前失效不会漏掉任何过期 TLB 项。
 ///
+/// A target that irreversibly stops cannot acknowledge. After observing its
+/// `Dead` publication with Acquire, the sender retires the pointer before
+/// ending the wait. `Dying` is insufficient: teardown still accesses memory.
+///
 /// x86_64 only; RISC-V's `send_ipi(FlushTLB, ..)` uses synchronous SBI `remote_sfence_vma`
 /// and does not need CSD.
 #[cfg(target_arch = "x86_64")]
 pub struct CsdFlushTlb {
     /// 指向本次请求的 `FlushTlbInfo`（发起方栈上对象）；`null` 表示无请求。
     info: AtomicPtr<FlushTlbInfo>,
-    /// 目标 CPU 完成失效后置 `true`。
+    /// Set after invalidation, or after retiring an irreversibly stopped target.
     done: AtomicBool,
 }
 
@@ -215,6 +222,23 @@ impl CsdFlushTlb {
         }
     }
 
+    /// Retire a request whose target can no longer consume or complete it.
+    ///
+    /// # Safety
+    /// The caller must guarantee no receiver can still consume or complete
+    /// this request. For a published per-CPU slot, this requires an Acquire
+    /// observation of irreversible `Dead` and publication serialization with
+    /// `FLUSH_TLB_GLOBAL_LOCK`. CPU stop must never resume ordinary execution,
+    /// including a suspended receiver. Supporting CPU restart requires a new
+    /// lifecycle handshake before reuse.
+    unsafe fn retire_dead_target(&self) {
+        // Clear even an unclaimed pointer before the sender's stack can unwind.
+        // A pointer already claimed by a terminally stopped receiver will never
+        // be dereferenced again; no live receiver may race this cancellation.
+        let _ = self.info.swap(core::ptr::null_mut(), Ordering::AcqRel);
+        self.done.store(true, Ordering::Release);
+    }
+
     /// 执行已取走的请求（调用方保证 `p` 在 `done` 置位前有效）。
     fn complete(&self, p: *const FlushTlbInfo, from_spin: bool) {
         // SAFETY: 发起方栈帧在 `done=true` 之前一直有效（见协议说明）。
@@ -482,15 +506,30 @@ pub fn remote_flush_tlb_on_ipi() {
 /// 1. Page table writes are complete and `inc_mm_tlb_gen` has been called.
 /// 2. `target_cpus` must not include this CPU (local flush is done by the caller after sending IPIs).
 ///
-/// Upon return, all CPUs in `target_cpus` have completed this shootdown (INV-4).
+/// On return, every selected CPU has acknowledged or irreversibly stopped,
+/// and no CSD retains `info` (INV-4).
 fn flush_tlb_multi(target_cpus: &AtomicCpuMask, info: &FlushTlbInfo) {
     // Filter out offline CPUs to avoid sending IPIs to not-yet-started / offline APICs.
-    // Capture online eligibility once. Every CSD preparation, IPI send and
+    // Capture eligibility once. Every CSD preparation, IPI send and
     // acknowledgement below uses this same private, immutable stack mask.
     let active = AtomicCpuMask::new();
     let my_cpu = smp_get_processor_id();
     for cpu in smp_cpu_manager().present_cpus().iter_cpu() {
-        if cpu != my_cpu && target_cpus.get(cpu) && smp_cpu_manager().is_online_cpu(cpu) {
+        if cpu == my_cpu || !target_cpus.get(cpu) {
+            continue;
+        }
+        // Dying x86 CPUs still execute teardown with IRQs disabled. Include
+        // them until Dead, servicing via lock-contention hooks or retiring the
+        // request after terminal shutdown. Other architectures use synchronous
+        // firmware invalidation and retain their existing online-only policy.
+        #[cfg(target_arch = "x86_64")]
+        let eligible = matches!(
+            smp_cpu_manager().cpuhp_state(cpu).state(),
+            CpuHpState::Online | CpuHpState::Dying
+        );
+        #[cfg(not(target_arch = "x86_64"))]
+        let eligible = smp_cpu_manager().is_online_cpu(cpu);
+        if eligible {
             active.set(cpu);
         }
     }
@@ -544,6 +583,13 @@ fn flush_tlb_multi(target_cpus: &AtomicCpuMask, info: &FlushTlbInfo) {
                 spins += 1;
                 if spins & 1023 == 0 {
                     service_pending_inbound_flush();
+                    if smp_cpu_manager().is_dead_cpu(cpu) {
+                        // Acquire observes terminal teardown. Dead cannot be
+                        // restarted by cpu_up(), and stop_this_cpu() will not
+                        // return to a receiver that could still use `info`.
+                        unsafe { csd.retire_dead_target() };
+                        break;
+                    }
                 }
                 #[cfg(feature = "mm_selftest")]
                 if let Some(observer) = &info.observer {
@@ -560,8 +606,8 @@ fn flush_tlb_multi(target_cpus: &AtomicCpuMask, info: &FlushTlbInfo) {
                 }
                 spin_loop();
             }
-            // No cleanup needed: the receiver already swapped the pointer out
-            // (`take`), so the slot is empty once `done` is observed.
+            // Both acknowledgement and terminal retirement leave the slot
+            // empty before the request's stack frame can be released.
         }
     }
 

kernel/src/mm/tlb/selftest.rs

@@ -608,18 +608,84 @@ fn run(
     passed
 }
 
+/// Exercise request retirement with a private slot, never falsifying a live
+/// CPU's lifecycle or installing pointers into the production per-CPU slots.
+fn csd_terminal_retirement_check() -> bool {
+    let slot = CsdFlushTlb::new();
+    let make_info = || FlushTlbInfo {
+        mm: None,
+        start: VirtAddr::new(0),
+        end: VirtAddr::new(0),
+        new_tlb_gen: 0,
+        stride_shift: MMArch::PAGE_SHIFT as u8,
+        freed_tables: false,
+        initiating_cpu: smp_get_processor_id(),
+        observer: None,
+    };
+    // No page-table operations are needed: an empty kernel range still runs
+    // the real claim/complete protocol, with no TlbState changes or invalidation.
+    let _irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() };
+    for claimed_before_stop in [false, true] {
+        {
+            let info = make_info();
+            slot.publish(&info);
+            if slot.done.load(Ordering::Acquire) {
+                return false;
+            }
+            if claimed_before_stop && slot.take() != Some(&info as *const FlushTlbInfo) {
+                return false;
+            }
+            // SAFETY: this fixture has no receiver at all, so no execution
+            // can resume with a claimed pointer. Publication is private.
+            unsafe { slot.retire_dead_target() };
+            if !slot.done.load(Ordering::Acquire) || !slot.info.load(Ordering::Acquire).is_null() {
+                return false;
+            }
+        }
+        // A delayed handler after the old stack object's lifetime must find
+        // nothing. Never dereference a pointer if cancellation was broken.
+        if slot.take().is_some() {
+            return false;
+        }
+        let next = make_info();
+        slot.publish(&next);
+        if slot.done.load(Ordering::Acquire) {
+            return false;
+        }
+        // A delayed old IPI may claim the new request: IPIs have no request
+        // identity. It must complete that request once, without a stale ack.
+        let Some(pointer) = slot.take() else {
+            return false;
+        };
+        if !core::ptr::eq(pointer, &next) || slot.take().is_some() {
+            return false;
+        }
+        slot.complete(pointer, false);
+        if !slot.done.load(Ordering::Acquire) || slot.take().is_some() {
+            return false;
+        }
+    }
+    true
+}
+
 fn run_report(target: ProcessorId) -> String {
     let _vmalloc_serial = crate::mm::vmalloc::lock_teardown_fixture();
     let mut report = String::new();
     use core::fmt::Write;
     let admission_ok = holder_admission_check();
+    let retirement_ok = csd_terminal_retirement_check();
     let _ = writeln!(report, "smp_available=ok");
     let _ = writeln!(
         report,
         "holder_admission={}",
         if admission_ok { "ok" } else { "fail" }
     );
-    let mut failures = usize::from(!admission_ok);
+    let _ = writeln!(
+        report,
+        "csd_terminal_retirement={}",
+        if retirement_ok { "ok" } else { "fail" }
+    );
+    let mut failures = usize::from(!admission_ok) + usize::from(!retirement_ok);
     for (user, lru, mode, key) in [
         (false, false, Mode::Normal, "kernel_teardown_order"),
         (

kernel/src/process/fork.rs

@@ -1353,9 +1353,11 @@ impl ProcessManager {
             drop(cpuset_guard);
 
             // 设置child_tid，意味着子线程能够知道自己的id。
-            // 按 Linux schedule_tail 语义，在子任务首次运行时再 best-effort 写入。
+            // Publish before the child is woken. The first user-return boundary
+            // writes with IRQs enabled, like Linux schedule_tail().
             if clone_flags.contains(CloneFlags::CLONE_CHILD_SETTID) {
                 pcb.thread.write_irqsave().set_child_tid = Some(clone_args.child_tid);
+                pcb.flags().insert(ProcessFlags::NEED_CHILD_TID);
             }
 
             // The shared user count was installed before registry publication.

kernel/src/process/manager/mod.rs

@@ -26,7 +26,6 @@ use crate::{
     },
     sched::{cpu_rq, enqueue_task_on_cpu, select_task_rq, OnRq, SchedClass, WakeupFlags},
     smp::{core::smp_get_processor_id, cpu::ProcessorId, kick_cpu},
-    syscall::user_access::write_one_to_user_protected,
 };
 
 mod exit;
@@ -707,14 +706,6 @@ impl ProcessManager {
         }
 
         crate::sched::notify_cpu_placement(&prev_pcb);
-        let set_child_tid = next_pcb.thread.write_irqsave().set_child_tid.take();
-        if let Some(addr) = set_child_tid {
-            // Align with Linux schedule_tail semantics: best-effort write of tid
-            // when the child task runs for the first time. Failure does not
-            // prevent the thread from continuing.
-            let child_tid = next_pcb.task_pid_vnr().data() as i32;
-            let _ = unsafe { write_one_to_user_protected(addr, &child_tid) };
-        }
     }
 
     /// If the target process is running on a remote CPU, force that CPU into