diff --git a/kernel/src/debug/mod.rs b/kernel/src/debug/mod.rs index fa63dd823..9c9fad650 100644 --- a/kernel/src/debug/mod.rs +++ b/kernel/src/debug/mod.rs @@ -14,6 +14,8 @@ pub mod page_cache; pub mod panic; pub mod rcu; pub mod rng; +#[cfg(feature = "sched_selftest")] +pub mod sched; pub mod sysfs; pub mod task_dump; pub mod timekeeping; diff --git a/kernel/src/debug/sched.rs b/kernel/src/debug/sched.rs new file mode 100644 index 000000000..be9355a90 --- /dev/null +++ b/kernel/src/debug/sched.rs @@ -0,0 +1,75 @@ +//! Scheduler user-return probes, present only in sched_selftest builds. + +use alloc::string::ToString; +use system_error::SystemError; + +use crate::{ + debug::sysfs::debugfs_kobj, + driver::base::kobject::KObject, + filesystem::{ + kernfs::callback::{KernCallbackData, KernFSCallback, KernFilePrivateData}, + vfs::{InodeMode, PollStatus}, + }, +}; + +#[derive(Debug)] +struct RseqNotifySelftest; + +impl KernFSCallback for RseqNotifySelftest { + fn open(&self, mut data: KernCallbackData) -> Result<(), SystemError> { + let report = crate::process::rseq::selftest::run()?; + data.file_private_data_mut() + .replace(KernFilePrivateData::DebugTextSnapshot(report)); + Ok(()) + } + + fn read( + &self, + data: KernCallbackData, + buf: &mut [u8], + offset: usize, + ) -> Result { + let report = match data.file_private_data() { + Some(KernFilePrivateData::DebugTextSnapshot(report)) => report, + _ => return Err(SystemError::EINVAL), + }; + let bytes = report.as_bytes(); + if offset >= bytes.len() { + return Ok(0); + } + let count = buf.len().min(bytes.len() - offset); + buf[..count].copy_from_slice(&bytes[offset..offset + count]); + Ok(count) + } + + fn write( + &self, + _data: KernCallbackData, + _buf: &[u8], + _offset: usize, + ) -> Result { + Err(SystemError::EPERM) + } + + fn poll(&self, _data: KernCallbackData) -> Result { + Ok(PollStatus::READ) + } +} + +pub fn init_debugfs_sched() -> Result<(), SystemError> { + let root = debugfs_kobj().inode().ok_or(SystemError::ENOENT)?; + let sched = root.add_dir( + "sched".to_string(), + InodeMode::from_bits_truncate(0o500), + None, + None, + )?; + sched.add_file( + "rseq_notify_selftest".to_string(), + InodeMode::S_IRUSR, + Some(256), + None, + Some(&RseqNotifySelftest), + )?; + Ok(()) +} diff --git a/kernel/src/debug/sysfs/mod.rs b/kernel/src/debug/sysfs/mod.rs index 8153a5107..56ca8529f 100644 --- a/kernel/src/debug/sysfs/mod.rs +++ b/kernel/src/debug/sysfs/mod.rs @@ -29,6 +29,8 @@ fn debugfs_init() -> Result<(), SystemError> { super::fuse::init_debugfs_fuse()?; super::kthread::init_debugfs_kthread()?; super::rng::init_debugfs_rng()?; + #[cfg(feature = "sched_selftest")] + super::sched::init_debugfs_sched()?; #[cfg(any(target_arch = "x86_64", target_arch = "riscv64"))] super::mm::init_debugfs_mm()?; super::page_cache::init_debugfs_page_cache()?; diff --git a/kernel/src/exception/entry.rs b/kernel/src/exception/entry.rs index 1533b6e47..5070fe24e 100644 --- a/kernel/src/exception/entry.rs +++ b/kernel/src/exception/entry.rs @@ -60,6 +60,21 @@ unsafe fn exit_to_user_mode_loop(frame: &mut TrapFrame, mut process_flags_work: if process_flags_work.contains(ProcessFlags::NEED_SCHEDULE) { schedule(SchedMode::SM_PREEMPT); } else { + if process_flags_work.contains(ProcessFlags::NEED_CHILD_TID) { + let pcb = ProcessManager::current_pcb(); + pcb.flags().remove(ProcessFlags::NEED_CHILD_TID); + let addr = pcb.take_set_child_tid(); + if let Some(addr) = addr { + let child_tid = pcb.task_pid_vnr().data() as i32; + // Like Linux schedule_tail(), write before delivering + // signals, outside the switch locks and with faults enabled. + CurrentIrqArch::interrupt_enable(); + let _ = + crate::syscall::user_access::write_one_to_user_protected(addr, &child_tid); + CurrentIrqArch::interrupt_disable(); + } + process_flags_work = pcb.flags().load(); + } if process_flags_work.contains(ProcessFlags::NEED_MEMORY_HIGH) { CurrentIrqArch::interrupt_enable(); crate::cgroup::memory::high::handle_user_return(); @@ -69,7 +84,14 @@ unsafe fn exit_to_user_mode_loop(frame: &mut TrapFrame, mut process_flags_work: // 优先处理 rseq，因为信号递送会保存 trapframe 到 sigframe // rseq 的 IP fixup 必须在信号递送之前完成 if process_flags_work.contains(ProcessFlags::NEED_RSEQ) { + // Consume before enabling IRQs: preemption/migration while + // touching user memory must leave a new notification pending. + ProcessManager::current_pcb() + .flags() + .remove(ProcessFlags::NEED_RSEQ); + CurrentIrqArch::interrupt_enable(); let _ = Rseq::handle_notify_resume(Some(frame)); + CurrentIrqArch::interrupt_disable(); process_flags_work = ProcessManager::current_pcb().flags().load(); } diff --git a/kernel/src/mm/tlb.rs b/kernel/src/mm/tlb.rs index 7ba8025bb..d67ba2406 100644 --- a/kernel/src/mm/tlb.rs +++ b/kernel/src/mm/tlb.rs @@ -11,7 +11,8 @@ //! - INV-1: At any point, if CPU `c`'s hardware page table equals `mm.table_paddr`, then bit `c` in `mm.active_cpus` is 1. //! - INV-2: `flush_tlb_*` must publish page table writes first, then `inc_mm_tlb_gen`, then read `active_cpus`. //! - INV-3: Freeing physical pages / page-table pages must happen after shootdown completion (guaranteed by `MmuGather`). -//! - INV-4: When `flush_tlb_multi` returns, all target CPUs have finished executing `local_flush_tlb_func`. +//! - INV-4: On return, each selected target has acknowledged invalidation or entered irreversible `Dead` state; +//! no CSD retains the request's stack pointer. //! - INV-5: When `freed_tables == true`, the receiving end must handle it (simplified: in this iteration the sender //! sends to all active CPUs regardless, and the receiver unconditionally executes). @@ -52,6 +53,8 @@ use crate::{ #[cfg(target_arch = "x86_64")] use crate::libs::spinlock::{SpinLock, SpinLockGuard}; +#[cfg(target_arch = "x86_64")] +use crate::smp::cpu::CpuHpState; /// Sentinel end value meaning "flush the entire mm"; combined with start=0 it means "do not use range-based flush". pub const TLB_FLUSH_ALL: VirtAddr = VirtAddr::new(usize::MAX); @@ -170,13 +173,17 @@ pub fn tlb_state_loaded_mm() -> Option> { /// 这在语义上仍然正确：页表写入发生在 `done=false`（Release）之前， /// handler 的 Acquire 读保证能看到新页表，提前失效不会漏掉任何过期 TLB 项。 /// +/// A target that irreversibly stops cannot acknowledge. After observing its +/// `Dead` publication with Acquire, the sender retires the pointer before +/// ending the wait. `Dying` is insufficient: teardown still accesses memory. +/// /// x86_64 only; RISC-V's `send_ipi(FlushTLB, ..)` uses synchronous SBI `remote_sfence_vma` /// and does not need CSD. #[cfg(target_arch = "x86_64")] pub struct CsdFlushTlb { /// 指向本次请求的 `FlushTlbInfo`（发起方栈上对象）；`null` 表示无请求。 info: AtomicPtr, - /// 目标 CPU 完成失效后置 `true`。 + /// Set after invalidation, or after retiring an irreversibly stopped target. done: AtomicBool, } @@ -215,6 +222,23 @@ impl CsdFlushTlb { } } + /// Retire a request whose target can no longer consume or complete it. + /// + /// # Safety + /// The caller must guarantee no receiver can still consume or complete + /// this request. For a published per-CPU slot, this requires an Acquire + /// observation of irreversible `Dead` and publication serialization with + /// `FLUSH_TLB_GLOBAL_LOCK`. CPU stop must never resume ordinary execution, + /// including a suspended receiver. Supporting CPU restart requires a new + /// lifecycle handshake before reuse. + unsafe fn retire_dead_target(&self) { + // Clear even an unclaimed pointer before the sender's stack can unwind. + // A pointer already claimed by a terminally stopped receiver will never + // be dereferenced again; no live receiver may race this cancellation. + let _ = self.info.swap(core::ptr::null_mut(), Ordering::AcqRel); + self.done.store(true, Ordering::Release); + } + /// 执行已取走的请求（调用方保证 `p` 在 `done` 置位前有效）。 fn complete(&self, p: *const FlushTlbInfo, from_spin: bool) { // SAFETY: 发起方栈帧在 `done=true` 之前一直有效（见协议说明）。 @@ -482,15 +506,30 @@ pub fn remote_flush_tlb_on_ipi() { /// 1. Page table writes are complete and `inc_mm_tlb_gen` has been called. /// 2. `target_cpus` must not include this CPU (local flush is done by the caller after sending IPIs). /// -/// Upon return, all CPUs in `target_cpus` have completed this shootdown (INV-4). +/// On return, every selected CPU has acknowledged or irreversibly stopped, +/// and no CSD retains `info` (INV-4). fn flush_tlb_multi(target_cpus: &AtomicCpuMask, info: &FlushTlbInfo) { // Filter out offline CPUs to avoid sending IPIs to not-yet-started / offline APICs. - // Capture online eligibility once. Every CSD preparation, IPI send and + // Capture eligibility once. Every CSD preparation, IPI send and // acknowledgement below uses this same private, immutable stack mask. let active = AtomicCpuMask::new(); let my_cpu = smp_get_processor_id(); for cpu in smp_cpu_manager().present_cpus().iter_cpu() { - if cpu != my_cpu && target_cpus.get(cpu) && smp_cpu_manager().is_online_cpu(cpu) { + if cpu == my_cpu || !target_cpus.get(cpu) { + continue; + } + // Dying x86 CPUs still execute teardown with IRQs disabled. Include + // them until Dead, servicing via lock-contention hooks or retiring the + // request after terminal shutdown. Other architectures use synchronous + // firmware invalidation and retain their existing online-only policy. + #[cfg(target_arch = "x86_64")] + let eligible = matches!( + smp_cpu_manager().cpuhp_state(cpu).state(), + CpuHpState::Online | CpuHpState::Dying + ); + #[cfg(not(target_arch = "x86_64"))] + let eligible = smp_cpu_manager().is_online_cpu(cpu); + if eligible { active.set(cpu); } } @@ -544,6 +583,13 @@ fn flush_tlb_multi(target_cpus: &AtomicCpuMask, info: &FlushTlbInfo) { spins += 1; if spins & 1023 == 0 { service_pending_inbound_flush(); + if smp_cpu_manager().is_dead_cpu(cpu) { + // Acquire observes terminal teardown. Dead cannot be + // restarted by cpu_up(), and stop_this_cpu() will not + // return to a receiver that could still use `info`. + unsafe { csd.retire_dead_target() }; + break; + } } #[cfg(feature = "mm_selftest")] if let Some(observer) = &info.observer { @@ -560,8 +606,8 @@ fn flush_tlb_multi(target_cpus: &AtomicCpuMask, info: &FlushTlbInfo) { } spin_loop(); } - // No cleanup needed: the receiver already swapped the pointer out - // (`take`), so the slot is empty once `done` is observed. + // Both acknowledgement and terminal retirement leave the slot + // empty before the request's stack frame can be released. } } diff --git a/kernel/src/mm/tlb/selftest.rs b/kernel/src/mm/tlb/selftest.rs index 45254d490..f8527e7fc 100644 --- a/kernel/src/mm/tlb/selftest.rs +++ b/kernel/src/mm/tlb/selftest.rs @@ -608,18 +608,84 @@ fn run( passed } +/// Exercise request retirement with a private slot, never falsifying a live +/// CPU's lifecycle or installing pointers into the production per-CPU slots. +fn csd_terminal_retirement_check() -> bool { + let slot = CsdFlushTlb::new(); + let make_info = || FlushTlbInfo { + mm: None, + start: VirtAddr::new(0), + end: VirtAddr::new(0), + new_tlb_gen: 0, + stride_shift: MMArch::PAGE_SHIFT as u8, + freed_tables: false, + initiating_cpu: smp_get_processor_id(), + observer: None, + }; + // No page-table operations are needed: an empty kernel range still runs + // the real claim/complete protocol, with no TlbState changes or invalidation. + let _irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; + for claimed_before_stop in [false, true] { + { + let info = make_info(); + slot.publish(&info); + if slot.done.load(Ordering::Acquire) { + return false; + } + if claimed_before_stop && slot.take() != Some(&info as *const FlushTlbInfo) { + return false; + } + // SAFETY: this fixture has no receiver at all, so no execution + // can resume with a claimed pointer. Publication is private. + unsafe { slot.retire_dead_target() }; + if !slot.done.load(Ordering::Acquire) || !slot.info.load(Ordering::Acquire).is_null() { + return false; + } + } + // A delayed handler after the old stack object's lifetime must find + // nothing. Never dereference a pointer if cancellation was broken. + if slot.take().is_some() { + return false; + } + let next = make_info(); + slot.publish(&next); + if slot.done.load(Ordering::Acquire) { + return false; + } + // A delayed old IPI may claim the new request: IPIs have no request + // identity. It must complete that request once, without a stale ack. + let Some(pointer) = slot.take() else { + return false; + }; + if !core::ptr::eq(pointer, &next) || slot.take().is_some() { + return false; + } + slot.complete(pointer, false); + if !slot.done.load(Ordering::Acquire) || slot.take().is_some() { + return false; + } + } + true +} + fn run_report(target: ProcessorId) -> String { let _vmalloc_serial = crate::mm::vmalloc::lock_teardown_fixture(); let mut report = String::new(); use core::fmt::Write; let admission_ok = holder_admission_check(); + let retirement_ok = csd_terminal_retirement_check(); let _ = writeln!(report, "smp_available=ok"); let _ = writeln!( report, "holder_admission={}", if admission_ok { "ok" } else { "fail" } ); - let mut failures = usize::from(!admission_ok); + let _ = writeln!( + report, + "csd_terminal_retirement={}", + if retirement_ok { "ok" } else { "fail" } + ); + let mut failures = usize::from(!admission_ok) + usize::from(!retirement_ok); for (user, lru, mode, key) in [ (false, false, Mode::Normal, "kernel_teardown_order"), ( diff --git a/kernel/src/process/fork.rs b/kernel/src/process/fork.rs index f41958cfc..c0ef768e6 100644 --- a/kernel/src/process/fork.rs +++ b/kernel/src/process/fork.rs @@ -1353,9 +1353,11 @@ impl ProcessManager { drop(cpuset_guard); // 设置child_tid，意味着子线程能够知道自己的id。 - // 按 Linux schedule_tail 语义，在子任务首次运行时再 best-effort 写入。 + // Publish before the child is woken. The first user-return boundary + // writes with IRQs enabled, like Linux schedule_tail(). if clone_flags.contains(CloneFlags::CLONE_CHILD_SETTID) { pcb.thread.write_irqsave().set_child_tid = Some(clone_args.child_tid); + pcb.flags().insert(ProcessFlags::NEED_CHILD_TID); } // The shared user count was installed before registry publication. diff --git a/kernel/src/process/manager/mod.rs b/kernel/src/process/manager/mod.rs index 804f9650f..22e277135 100644 --- a/kernel/src/process/manager/mod.rs +++ b/kernel/src/process/manager/mod.rs @@ -26,7 +26,6 @@ use crate::{ }, sched::{cpu_rq, enqueue_task_on_cpu, select_task_rq, OnRq, SchedClass, WakeupFlags}, smp::{core::smp_get_processor_id, cpu::ProcessorId, kick_cpu}, - syscall::user_access::write_one_to_user_protected, }; mod exit; @@ -707,14 +706,6 @@ impl ProcessManager { } crate::sched::notify_cpu_placement(&prev_pcb); - let set_child_tid = next_pcb.thread.write_irqsave().set_child_tid.take(); - if let Some(addr) = set_child_tid { - // Align with Linux schedule_tail semantics: best-effort write of tid - // when the child task runs for the first time. Failure does not - // prevent the thread from continuing. - let child_tid = next_pcb.task_pid_vnr().data() as i32; - let _ = unsafe { write_one_to_user_protected(addr, &child_tid) }; - } } /// If the target process is running on a remote CPU, force that CPU into diff --git a/kernel/src/process/rseq.rs b/kernel/src/process/rseq.rs index d29000234..ff3c92c11 100644 --- a/kernel/src/process/rseq.rs +++ b/kernel/src/process/rseq.rs @@ -12,6 +12,9 @@ //! //! 参考: Linux 6.6.21 kernel/rseq.c +#[cfg(feature = "sched_selftest")] +pub(crate) mod selftest; + use core::sync::atomic::{AtomicU32, Ordering}; use system_error::SystemError; @@ -557,6 +560,10 @@ impl Rseq { /// 处理 notify-resume /// /// 在返回用户态前调用，执行 IP 修正和 cpu_id 更新 + /// + /// The user-return caller consumes NEED_RSEQ before calling with IRQs + /// enabled. Never clear it here: a preemption/migration during user access + /// requires another pass, including when called from signal delivery. pub fn handle_notify_resume(frame: Option<&mut F>) -> Result<(), ()> { use crate::{arch::MMArch, mm::MemoryManagementArch}; @@ -572,7 +579,6 @@ impl Rseq { match rseq_state.registration() { Some(reg) => (reg.ptr, reg.sig), None => { - pcb.flags().remove(ProcessFlags::NEED_RSEQ); return Ok(()); } } @@ -585,7 +591,6 @@ impl Rseq { if let Some(frame) = frame { if let Err(e) = Self::ip_fixup(frame, &access, sig, user_end, &pcb) { log::debug!("rseq ip_fixup failed: {:?}", e); - pcb.flags().remove(ProcessFlags::NEED_RSEQ); let _ = crate::ipc::signal::force_kernel_signal_to_current( crate::arch::ipc::signal::Signal::SIGSEGV, ); @@ -595,16 +600,16 @@ impl Rseq { // 更新 cpu_id 等字段 let cpu_id = smp_get_processor_id().data() as u32; + #[cfg(feature = "sched_selftest")] + selftest::after_cpu_id_capture(&pcb); if let Err(_e) = unsafe { access.update_cpu_node_id(cpu_id, 0, 0) } { // log::debug!("rseq update_cpu_node_id failed: {:?}", e); - pcb.flags().remove(ProcessFlags::NEED_RSEQ); let _ = crate::ipc::signal::force_kernel_signal_to_current( crate::arch::ipc::signal::Signal::SIGSEGV, ); return Err(()); } - pcb.flags().remove(ProcessFlags::NEED_RSEQ); Ok(()) } diff --git a/kernel/src/process/rseq/selftest.rs b/kernel/src/process/rseq/selftest.rs new file mode 100644 index 000000000..6012aab62 --- /dev/null +++ b/kernel/src/process/rseq/selftest.rs @@ -0,0 +1,76 @@ +//! Re-publish a normal migration notification inside the real faultable handler. +//! This tests notification ownership, without pretending a CPU migration occurred. + +use alloc::{string::String, sync::Arc}; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use system_error::SystemError; + +use crate::{ + arch::{interrupt::TrapFrame, CurrentIrqArch}, + exception::InterruptArch, + process::{ProcessControlBlock, ProcessFlags, ProcessManager}, +}; + +use super::Rseq; + +static OWNER: AtomicUsize = AtomicUsize::new(0); +static ARMED: AtomicBool = AtomicBool::new(false); + +struct Probe; + +impl Drop for Probe { + fn drop(&mut self) { + ARMED.store(false, Ordering::Relaxed); + OWNER.store(0, Ordering::Release); + } +} + +pub(super) fn after_cpu_id_capture(pcb: &Arc) { + let tag = pcb.raw_pid().data() + 1; + if OWNER.load(Ordering::Acquire) == tag && ARMED.swap(false, Ordering::AcqRel) { + // Use the scheduler's event producer, not a raw process-flag store. + Rseq::on_migrate(pcb); + } +} + +pub(crate) fn run() -> Result { + let pcb = ProcessManager::current_pcb(); + if !pcb.rseq_state().is_registered() { + return Err(SystemError::EINVAL); + } + OWNER + .compare_exchange( + 0, + pcb.raw_pid().data() + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) + .map_err(|_| SystemError::EBUSY)?; + let _probe = Probe; + ARMED.store(true, Ordering::Release); + + // Match the return boundary's notification consumption and IRQ context. + // The user registration remains installed and both calls access real user + // memory. No locks cover those accesses. + let irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() }; + pcb.flags().remove(ProcessFlags::NEED_RSEQ); + unsafe { CurrentIrqArch::interrupt_enable() }; + let first = Rseq::handle_notify_resume::(None); + unsafe { CurrentIrqArch::interrupt_disable() }; + let injected = !ARMED.load(Ordering::Acquire); + let preserved = pcb.flags().test_and_clear(ProcessFlags::NEED_RSEQ); + unsafe { CurrentIrqArch::interrupt_enable() }; + let second = Rseq::handle_notify_resume::(None); + unsafe { CurrentIrqArch::interrupt_disable() }; + drop(irq_guard); + + // A real preemption during the second pass may leave another notification; + // do not consume it or require the final flag to be empty. + let ok = injected && preserved && first.is_ok() && second.is_ok(); + Ok(alloc::format!( + "status={}\nmigration_notification_injected={}\nnotification_preserved={}\nhandler_passes=2\n", + if ok { "ok" } else { "fail" }, + injected, + preserved, + )) +} diff --git a/kernel/src/process/state.rs b/kernel/src/process/state.rs index 91538a360..4c62f6185 100644 --- a/kernel/src/process/state.rs +++ b/kernel/src/process/state.rs @@ -233,6 +233,8 @@ bitflags! { /// Process is waiting for an I/O operation to complete (used for iowait /// accounting). const IN_IOWAIT = 1 << 13; + /// First user return must perform the best-effort CLONE_CHILD_SETTID write. + const NEED_CHILD_TID = 1 << 14; /// PID links and visible-thread accounting have already been released. const PID_UNHASHED = 1 << 15; /// Task is currently traced by another task. @@ -293,6 +295,7 @@ impl ProcessFlags { | Self::NEED_MEMORY_HIGH.bits | Self::NEED_CGROUP_FREEZE.bits | Self::PENDING_GROUP_STOP.bits + | Self::NEED_CHILD_TID.bits } /// Test and clear flags. diff --git a/kernel/src/process/task.rs b/kernel/src/process/task.rs index 7edbd7718..b3d9c6382 100644 --- a/kernel/src/process/task.rs +++ b/kernel/src/process/task.rs @@ -2065,6 +2065,12 @@ impl ProcessControlBlock { self.rseq_state.write_irqsave() } + /// Consume the first-run child TID address before faultable user access. + #[inline] + pub(crate) fn take_set_child_tid(&self) -> Option { + self.thread.write_irqsave().set_child_tid.take() + } + pub fn try_siginfo_mut(&self, times: u8) -> Option> { for _ in 0..times { if let Some(r) = self.sig_info.try_write_irqsave() { diff --git a/user/apps/tests/dunitest/no_skip.txt b/user/apps/tests/dunitest/no_skip.txt index ee54b228d..ff78170a9 100644 --- a/user/apps/tests/dunitest/no_skip.txt +++ b/user/apps/tests/dunitest/no_skip.txt @@ -73,3 +73,5 @@ normal/brk_semantics normal/mmap_fixed_address normal/rlimit_data normal/fork_mapping_holes + +normal/user_return_work diff --git a/user/apps/tests/dunitest/suites/normal/test_tlb_shootdown.cc b/user/apps/tests/dunitest/suites/normal/test_tlb_shootdown.cc index 865563fd2..a53367477 100644 --- a/user/apps/tests/dunitest/suites/normal/test_tlb_shootdown.cc +++ b/user/apps/tests/dunitest/suites/normal/test_tlb_shootdown.cc @@ -596,7 +596,7 @@ TEST(TlbShootdown, TeardownRetainsRemoteHolderUntilAcknowledged) { EXPECT_NE(std::string::npos, report.find("status=ok\n")) << report; EXPECT_NE(std::string::npos, report.find("summary_fail=0\n")) << report; for (const char* key : { - "smp_available", "holder_admission", + "smp_available", "holder_admission", "csd_terminal_retirement", "kernel_teardown_order", "kernel_missing_flush_detected", "kernel_missing_ack_detected", "user_teardown_order", "user_missing_flush_detected", "user_missing_ack_detected", "user_lru_teardown_order", "user_lru_missing_flush_detected", "user_lru_missing_ack_detected", "user_multibatch_teardown_order"}) { diff --git a/user/apps/tests/dunitest/suites/normal/user_return_work.cc b/user/apps/tests/dunitest/suites/normal/user_return_work.cc new file mode 100644 index 000000000..97309fa76 --- /dev/null +++ b/user/apps/tests/dunitest/suites/normal/user_return_work.cc @@ -0,0 +1,186 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +long CloneChildTid(pid_t* child_tid) { + struct clone_args args = {}; + args.flags = CLONE_CHILD_SETTID; + args.exit_signal = SIGCHLD; + args.child_tid = reinterpret_cast(child_tid); + return syscall(SYS_clone3, &args, sizeof(args)); +} + +void ExpectChildSuccess(pid_t child) { + int status = 0; + ASSERT_EQ(waitpid(child, &status, 0), child) << strerror(errno); + ASSERT_TRUE(WIFEXITED(status)) << "status=" << status; + EXPECT_EQ(WEXITSTATUS(status), 0); +} + +#if defined(__x86_64__) +extern "C" { +extern const ptrdiff_t __rseq_offset __attribute__((weak)); +extern const unsigned int __rseq_size __attribute__((weak)); +} + +struct alignas(32) RseqAbi { + uint32_t cpu_id_start; + uint32_t cpu_id; + uint64_t rseq_cs; + uint32_t flags; + uint32_t node_id; + uint32_t mm_cid; + uint32_t padding; +}; +constexpr uint32_t kRseqSignature = 0x53053053; + +// Only call after fork: changing the test runner's libc registration would +// invalidate libc's own assumptions about its TLS region. +RseqAbi* RegisterPrivateRseq() { + if (&__rseq_size != nullptr && &__rseq_offset != nullptr && __rseq_size != 0) { + void* libc_rseq = static_cast(__builtin_thread_pointer()) + __rseq_offset; + if (syscall(SYS_rseq, libc_rseq, sizeof(RseqAbi), 1, kRseqSignature) != 0) { + return nullptr; + } + } + void* page = mmap(nullptr, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (page == MAP_FAILED) return nullptr; + auto* rseq = static_cast(page); + rseq->cpu_id_start = rseq->cpu_id = UINT32_MAX; + if (syscall(SYS_rseq, rseq, sizeof(*rseq), 0, kRseqSignature) != 0) return nullptr; + return rseq; +} +#endif + +} // namespace + +TEST(UserReturnWork, ChildTidIsWrittenBeforeFirstUserInstructionWithCow) { + auto* tid = static_cast(mmap(nullptr, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + ASSERT_NE(tid, MAP_FAILED); + constexpr pid_t kSentinel = -123; + for (int iteration = 0; iteration < 16; ++iteration) { + *tid = kSentinel; + long child = CloneChildTid(tid); + ASSERT_GE(child, 0) << strerror(errno); + if (child == 0) { + // Read before entering another syscall which could mask a late write. + pid_t observed = *static_cast(tid); + _exit(observed == syscall(SYS_gettid) ? 0 : 1); + } + ExpectChildSuccess(child); + EXPECT_EQ(*tid, kSentinel); + } + EXPECT_EQ(munmap(tid, 4096), 0); +} + +TEST(UserReturnWork, InvalidChildTidIsBestEffort) { + long child = CloneChildTid(nullptr); + ASSERT_GE(child, 0) << strerror(errno); + if (child == 0) _exit(0); + ExpectChildSuccess(child); +} + +TEST(UserReturnWork, InaccessibleChildTidDoesNotKillChild) { + void* page = mmap(nullptr, 4096, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(page, MAP_FAILED); + long child = CloneChildTid(static_cast(page)); + ASSERT_GE(child, 0) << strerror(errno); + if (child == 0) _exit(0); + ExpectChildSuccess(child); + EXPECT_EQ(munmap(page, 4096), 0); +} + +#if defined(__x86_64__) +TEST(UserReturnWork, RseqInheritedPrivatePageRemainsUsableAfterFork) { + pid_t worker = fork(); + ASSERT_GE(worker, 0) << strerror(errno); + if (worker == 0) { + alarm(20); + RseqAbi* rseq = RegisterPrivateRseq(); + if (rseq == nullptr) _exit(10); + cpu_set_t allowed; + CPU_ZERO(&allowed); + if (sched_getaffinity(0, sizeof(allowed), &allowed) != 0) _exit(11); + int cpus[2] = {-1, -1}; + for (int cpu = 0; cpu < CPU_SETSIZE; ++cpu) { + if (CPU_ISSET(cpu, &allowed)) { + if (cpus[0] < 0) cpus[0] = cpu; + else { cpus[1] = cpu; break; } + } + } + if (cpus[0] < 0) _exit(12); + if (cpus[1] < 0) cpus[1] = cpus[0]; + for (int iteration = 0; iteration < 16; ++iteration) { + const int cpu = cpus[iteration % 2]; + cpu_set_t pinned; + CPU_ZERO(&pinned); + CPU_SET(cpu, &pinned); + if (sched_setaffinity(0, sizeof(pinned), &pinned) != 0) _exit(13); + if (rseq->cpu_id != static_cast(cpu)) _exit(14); + pid_t child = fork(); + if (child < 0) _exit(15); + if (child == 0) { + const uint32_t start = *static_cast(&rseq->cpu_id_start); + const uint32_t current = *static_cast(&rseq->cpu_id); + _exit(start == static_cast(cpu) && current == start ? 0 : 16); + } + int status = 0; + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) _exit(17); + } + if (syscall(SYS_rseq, rseq, sizeof(*rseq), 1, kRseqSignature) != 0) _exit(18); + _exit(0); + } + ExpectChildSuccess(worker); +} + +TEST(UserReturnWork, MigrationNotificationSurvivesRealResumeHandler) { + constexpr const char* path = "/sys/kernel/debug/sched/rseq_notify_selftest"; + if (access(path, R_OK) != 0 && errno == ENOENT) { + GTEST_SKIP() << "requires a sched_selftest kernel"; + } + pid_t worker = fork(); + ASSERT_GE(worker, 0) << strerror(errno); + if (worker == 0) { + alarm(20); + RseqAbi* rseq = RegisterPrivateRseq(); + if (rseq == nullptr) _exit(20); + int fd = open(path, O_RDONLY); + if (fd < 0) _exit(21); + char report[256] = {}; + ssize_t count = read(fd, report, sizeof(report) - 1); + close(fd); + if (count <= 0) _exit(22); + // Check actual handler execution and notification preservation. + if (strstr(report, "status=ok\n") == nullptr || + strstr(report, "migration_notification_injected=true\n") == nullptr || + strstr(report, "notification_preserved=true\n") == nullptr || + strstr(report, "handler_passes=2\n") == nullptr) { + if (write(STDERR_FILENO, report, count) < 0) _exit(25); + _exit(23); + } + if (syscall(SYS_rseq, rseq, sizeof(*rseq), 1, kRseqSignature) != 0) _exit(24); + _exit(0); + } + ExpectChildSuccess(worker); +} +#endif + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/user/apps/tests/dunitest/whitelist.txt b/user/apps/tests/dunitest/whitelist.txt index 27ed3c3ff..f591a9559 100644 --- a/user/apps/tests/dunitest/whitelist.txt +++ b/user/apps/tests/dunitest/whitelist.txt @@ -319,3 +319,5 @@ normal/brk_semantics normal/mmap_fixed_address normal/rlimit_data normal/fork_mapping_holes + +normal/user_return_work