From 052a1465e896e63e10b283dbab6bf6909bbd9d73 Mon Sep 17 00:00:00 2001 From: Ryan Breen Date: Thu, 19 Feb 2026 05:16:29 -0500 Subject: [PATCH 1/2] fix: ARM64 TPIDR_EL0 context switch save/restore and signal trampoline syscall number Two ARM64 fixes that resolve DATA_ABORT crashes at FAR=0xffffffffffffff5c when running BusyBox ash shell and musl-linked programs: 1. TPIDR_EL0 (Thread Local Storage pointer) was never saved or restored during context switches. When multiple userspace processes share a CPU, they clobber each other's TLS pointer. musl libc stores errno and pthread structures via TPIDR_EL0, so a zeroed/wrong value causes accesses to addresses like 0 - 164 = 0xffffffffffffff5c. Added tpidr_el0 field to CpuContext and mrs/msr instructions in all 5 context switch paths: save_userspace, save_kernel, restore_kernel, restore_userspace, and setup_first_entry. Also preserved in from_aarch64_frame for fork inheritance. 2. Signal trampoline used x86_64 syscall number 15 for rt_sigreturn instead of the correct ARM64 number 139. This caused "Unknown syscall" errors when signal handlers returned via the trampoline. Co-Authored-By: Claude Opus 4.6 --- .../src/arch_impl/aarch64/context_switch.rs | 40 +++++++++++++++++++ kernel/src/signal/trampoline.rs | 9 +++-- kernel/src/task/thread.rs | 10 +++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/kernel/src/arch_impl/aarch64/context_switch.rs b/kernel/src/arch_impl/aarch64/context_switch.rs index 19c939cd..62444ca7 100644 --- a/kernel/src/arch_impl/aarch64/context_switch.rs +++ b/kernel/src/arch_impl/aarch64/context_switch.rs @@ -293,6 +293,13 @@ fn save_userspace_context_inline(thread: &mut Thread, frame: &Aarch64ExceptionFr } thread.context.sp_el0 = sp_el0; + // Save TPIDR_EL0 (user TLS pointer) - critical for musl/libc TLS correctness + let tpidr: u64; + unsafe { + core::arch::asm!("mrs {}, tpidr_el0", out(reg) tpidr, options(nomem, nostack)); + } + thread.context.tpidr_el0 = tpidr; + // CRITICAL: Save kernel stack pointer for blocked-in-syscall restoration. thread.context.sp = frame as *const _ as u64 + 272; } @@ -351,6 +358,13 @@ fn save_kernel_context_inline(thread: &mut Thread, frame: &Aarch64ExceptionFrame core::arch::asm!("mrs {}, sp_el0", out(reg) sp_el0, options(nomem, nostack)); } thread.context.sp_el0 = sp_el0; + + // Save TPIDR_EL0 (user TLS pointer) - critical for musl/libc TLS correctness + let tpidr: u64; + unsafe { + core::arch::asm!("mrs {}, tpidr_el0", out(reg) tpidr, options(nomem, nostack)); + } + thread.context.tpidr_el0 = tpidr; } /// Restore kernel thread context into frame — called inside scheduler lock hold. @@ -477,6 +491,15 @@ fn restore_kernel_context_inline( } } + // Restore TPIDR_EL0 (user TLS pointer) - critical for musl/libc TLS correctness + unsafe { + core::arch::asm!( + "msr tpidr_el0, {}", + in(reg) thread.context.tpidr_el0, + options(nomem, nostack) + ); + } + // Memory barrier to ensure all writes are visible core::sync::atomic::fence(Ordering::SeqCst); true @@ -528,6 +551,15 @@ fn restore_userspace_context_inline(thread: &mut Thread, frame: &mut Aarch64Exce options(nomem, nostack) ); } + + // Restore TPIDR_EL0 (user TLS pointer) - critical for musl/libc TLS correctness + unsafe { + core::arch::asm!( + "msr tpidr_el0, {}", + in(reg) thread.context.tpidr_el0, + options(nomem, nostack) + ); + } } /// Set up first userspace entry — called inside scheduler lock hold. @@ -579,6 +611,14 @@ fn setup_first_entry_inline(thread: &mut Thread, frame: &mut Aarch64ExceptionFra frame.x28 = 0; frame.x29 = 0; frame.x30 = 0; + + // Clear TPIDR_EL0 - musl will set it during __init_tls + unsafe { + core::arch::asm!( + "msr tpidr_el0, xzr", + options(nomem, nostack) + ); + } } // ============================================================================= diff --git a/kernel/src/signal/trampoline.rs b/kernel/src/signal/trampoline.rs index 402dd70f..45a98835 100644 --- a/kernel/src/signal/trampoline.rs +++ b/kernel/src/signal/trampoline.rs @@ -38,20 +38,23 @@ pub const SIGNAL_TRAMPOLINE_SIZE: usize = SIGNAL_TRAMPOLINE.len(); /// This is the raw machine code that will be executed in userspace. /// /// Assembly (little-endian ARM64): -/// mov x8, #15 ; SYS_rt_sigreturn (syscall number 15) +/// mov x8, #139 ; SYS_rt_sigreturn (aarch64 syscall number 139) /// svc #0 ; Trigger syscall /// brk #1 ; Should never reach here (causes debug exception if it does) /// /// On ARM64, the signal handler returns via BLR/RET to x30 (link register), /// which we set to point to this trampoline. /// +/// Note: ARM64 uses asm-generic syscall numbers, NOT x86_64 numbers. +/// rt_sigreturn is 139 on ARM64, not 15 as on x86_64. +/// /// Instruction encoding (little-endian): -/// - mov x8, #15: 0xD28001E8 -> E8 01 80 D2 +/// - mov x8, #139: 0xD2801168 -> 68 11 80 D2 /// - svc #0: 0xD4000001 -> 01 00 00 D4 /// - brk #1: 0xD4200020 -> 20 00 20 D4 #[cfg(target_arch = "aarch64")] pub static SIGNAL_TRAMPOLINE: [u8; 12] = [ - 0xE8, 0x01, 0x80, 0xD2, // mov x8, #15 (rt_sigreturn syscall number) + 0x68, 0x11, 0x80, 0xD2, // mov x8, #139 (rt_sigreturn - aarch64 syscall number) 0x01, 0x00, 0x00, 0xD4, // svc #0 (supervisor call - trigger syscall) 0x20, 0x00, 0x20, 0xD4, // brk #1 (should never reach here) ]; diff --git a/kernel/src/task/thread.rs b/kernel/src/task/thread.rs index f903689e..01ddf714 100644 --- a/kernel/src/task/thread.rs +++ b/kernel/src/task/thread.rs @@ -236,6 +236,8 @@ pub struct CpuContext { pub elr_el1: u64, /// Saved program status (includes EL0 mode bits) pub spsr_el1: u64, + /// Thread pointer (TPIDR_EL0) - used by musl/libc for Thread Local Storage + pub tpidr_el0: u64, } #[cfg(target_arch = "aarch64")] @@ -268,6 +270,7 @@ impl CpuContext { elr_el1: 0, // SPSR with EL1h mode, interrupts masked initially spsr_el1: 0x3c5, // EL1h, DAIF masked + tpidr_el0: 0, } } @@ -295,6 +298,7 @@ impl CpuContext { elr_el1: entry_point, // Where to jump in userspace // SPSR for EL0: mode=0 (EL0t), DAIF clear (interrupts enabled) spsr_el1: 0x0, // EL0t with interrupts enabled + tpidr_el0: 0, // TLS pointer, set by musl during __init_tls } } @@ -303,6 +307,11 @@ impl CpuContext { /// This captures the userspace context from the exception frame saved by the syscall entry. /// The exception frame contains all registers as they were at the time of the SVC instruction. pub fn from_aarch64_frame(frame: &crate::arch_impl::aarch64::exception_frame::Aarch64ExceptionFrame, user_sp: u64) -> Self { + // Read TPIDR_EL0 (user TLS pointer) so forked children inherit it + let tpidr: u64; + unsafe { + core::arch::asm!("mrs {}, tpidr_el0", out(reg) tpidr, options(nomem, nostack)); + } Self { // All general-purpose registers from the exception frame x0: frame.x0, @@ -340,6 +349,7 @@ impl CpuContext { sp_el0: user_sp, // User stack pointer (passed separately since it's in SP_EL0) elr_el1: frame.elr, // Return address (where to resume after syscall) spsr_el1: frame.spsr, // Saved program status + tpidr_el0: tpidr, // User TLS pointer (inherited by forked child) } } } From de2f3ee7ce665c39a83a55ccf858feb21a2fd653 Mon Sep 17 00:00:00 2001 From: Ryan Breen Date: Thu, 19 Feb 2026 06:28:14 -0500 Subject: [PATCH 2/2] fix: /dev/tty controlling terminal lookup by session ID, TIOCSCTTY job control setup Three fixes for BusyBox ash "can't access tty; job control turned off": 1. /dev/tty open now matches by session ID instead of PID. The controlling terminal belongs to the entire session, not just the session leader. When ash (child of bsh) opens /dev/tty, it needs to find the PTY that bsh (the session leader) set up via TIOCSCTTY. Previously matched by PID which failed for any non-session-leader process. 2. TIOCSCTTY now also sets foreground_pgid so tcgetpgrp() returns a valid value instead of 0. 3. bwm's spawn_child calls TIOCSCTTY after opening PTY slave, and libbreenix gains set_controlling_terminal() wrapper. Root cause confirmed by independent Codex + Opus investigation (panel of experts pattern). Co-Authored-By: Claude Opus 4.6 --- kernel/src/syscall/fs.rs | 46 ++++++++++++++++++++++++++++++++++ kernel/src/tty/ioctl.rs | 9 ++++++- libs/libbreenix/src/termios.rs | 12 +++++++++ userspace/programs/src/bwm.rs | 5 ++++ 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/kernel/src/syscall/fs.rs b/kernel/src/syscall/fs.rs index 118655e6..c3c15e78 100644 --- a/kernel/src/syscall/fs.rs +++ b/kernel/src/syscall/fs.rs @@ -2179,6 +2179,52 @@ fn handle_devfs_open(device_name: &str, _flags: u32) -> SyscallResult { } }; + // For /dev/tty, redirect to the controlling PTY if one exists. + // On Linux, /dev/tty is a magic device that refers to the calling process's + // controlling terminal. The controlling terminal belongs to the SESSION, + // not a single process. Any process in the session can open /dev/tty. + if matches!(device.device_type, devfs::DeviceType::Tty) { + let sid = process.sid.as_u64() as u32; + // Drop manager lock before accessing PTY subsystem to avoid lock ordering issues + drop(manager_guard); + + // Search active PTYs for one controlled by this session. + // controlling_pid stores the session leader's PID (== SID). + for pty_num in crate::tty::pty::list_active() { + if let Some(pair) = crate::tty::pty::get(pty_num) { + if pair.controlling_pid.lock().map_or(false, |p| p == sid) { + // Found the controlling PTY — open as PtySlave + let thread_id2 = crate::task::scheduler::current_thread_id().unwrap(); + let mut mg = crate::process::manager(); + let proc2 = mg.as_mut().unwrap() + .find_process_by_thread_mut(thread_id2).unwrap().1; + let fd_kind = FdKind::PtySlave(pty_num); + return match proc2.fd_table.alloc(fd_kind) { + Ok(fd) => { + log::info!("handle_devfs_open: /dev/tty -> PTY slave {} as fd {}", pty_num, fd); + SyscallResult::Ok(fd as u64) + } + Err(_) => SyscallResult::Err(EMFILE as u64), + }; + } + } + } + // No controlling terminal found — fall through to generic device + // Re-acquire manager lock for the generic path + let thread_id2 = crate::task::scheduler::current_thread_id().unwrap(); + let mut manager_guard = crate::process::manager(); + let process = manager_guard.as_mut().unwrap() + .find_process_by_thread_mut(thread_id2).unwrap().1; + let fd_kind = FdKind::Device(device.device_type); + return match process.fd_table.alloc(fd_kind) { + Ok(fd) => { + log::info!("handle_devfs_open: /dev/tty (no ctty) as fd {}", fd); + SyscallResult::Ok(fd as u64) + } + Err(_) => SyscallResult::Err(EMFILE as u64), + }; + } + // Allocate file descriptor with Device kind let fd_kind = FdKind::Device(device.device_type); match process.fd_table.alloc(fd_kind) { diff --git a/kernel/src/tty/ioctl.rs b/kernel/src/tty/ioctl.rs index 1d98f1c8..263b20b9 100644 --- a/kernel/src/tty/ioctl.rs +++ b/kernel/src/tty/ioctl.rs @@ -383,7 +383,14 @@ pub fn handle_tiocsctty(pair: &Arc, arg: u64, pid: u32) -> Result<(), i } *controlling = Some(pid); - log::debug!("PTY{}: Set controlling process to {}", pair.pty_num, pid); + + // Also set the foreground process group to the caller's PGID. + // On Linux, TIOCSCTTY auto-sets the foreground pgrp so that tcgetpgrp() + // returns a valid value. Without this, shells like ash fail job control + // setup because tcgetpgrp() returns 0. + *pair.foreground_pgid.lock() = Some(pid); + + log::debug!("PTY{}: Set controlling process to {}, foreground pgid to {}", pair.pty_num, pid, pid); Ok(()) } diff --git a/libs/libbreenix/src/termios.rs b/libs/libbreenix/src/termios.rs index df4afaf0..53579bbb 100644 --- a/libs/libbreenix/src/termios.rs +++ b/libs/libbreenix/src/termios.rs @@ -18,6 +18,7 @@ pub mod request { pub const TCSETSF: u64 = 0x5404; pub const TIOCGPGRP: u64 = 0x540F; pub const TIOCSPGRP: u64 = 0x5410; + pub const TIOCSCTTY: u64 = 0x540E; pub const TIOCGWINSZ: u64 = 0x5413; pub const TIOCSWINSZ: u64 = 0x5414; } @@ -201,6 +202,17 @@ pub fn tcsetpgrp(fd: Fd, pgrp: i32) -> Result<(), Error> { Error::from_syscall(ret as i64).map(|_| ()) } +/// Set the controlling terminal for the current session (TIOCSCTTY) +/// +/// Must be called after setsid() and opening a PTY slave. +/// This makes the PTY the controlling terminal, enabling job control. +pub fn set_controlling_terminal(fd: Fd) -> Result<(), Error> { + let ret = unsafe { + raw::syscall3(nr::IOCTL, fd.raw(), request::TIOCSCTTY, 0) + }; + Error::from_syscall(ret as i64).map(|_| ()) +} + /// Make raw mode termios settings pub fn cfmakeraw(termios: &mut Termios) { termios.c_iflag &= !(iflag::ICRNL | iflag::IXON); diff --git a/userspace/programs/src/bwm.rs b/userspace/programs/src/bwm.rs index 77a9e6ff..4c3d03db 100644 --- a/userspace/programs/src/bwm.rs +++ b/userspace/programs/src/bwm.rs @@ -695,6 +695,11 @@ fn spawn_child(path: &[u8], _name: &str) -> (Fd, i64) { } }; + // Set the slave PTY as the controlling terminal for this session. + // Without this, shells like ash can't set up job control and print + // "can't access tty; job control turned off". + let _ = libbreenix::termios::set_controlling_terminal(slave_fd); + // Dup to stdin/stdout/stderr let _ = io::dup2(slave_fd, Fd::from_raw(0)); let _ = io::dup2(slave_fd, Fd::from_raw(1));