Access kernel stack using only KSEG2 addresses
MODIFY
include/arm64/types.h
include/arm64/types.h
#ifndef _ARM64_TYPES_H_
#define _ARM64_TYPES_H_
+#include <stdint.h>
+
typedef int64_t register_t;
typedef int64_t fpregister_t;

MODIFY
include/mips/pcpu.h
include/mips/pcpu.h
#ifndef _MIPS_PCPU_H_
#define _MIPS_PCPU_H_
-#ifndef _MACHDEP
-#error "Do not use this header file outside kernel machine dependent code!"
-#endif
+#define PCPU_MD_FIELDS \
+ struct { \
+ /*!< kernel sp restored on user->kernel transition */ \
+ void *ksp; \
+ /*!< registers that cannot be saved directly to kernel stack */ \
+ register_t status, sp, cause, epc, badvaddr; \
+ }
+#ifdef _MACHDEP
#ifdef __ASSEMBLER__
#include <mips/asm.h>
#include <mips/mips.h>
#define LOAD_PCPU(reg) LA reg, _pcpu_data
-#define LOAD_PCPU_KSEG0(reg) LA reg, MIPS_KSEG2_TO_KSEG0(_pcpu_data)
#endif /* !__ASSEMBLER__ */
+#endif /* !_MACHDEP */
#endif /* !_MIPS_PCPU_H_ */

MODIFY
include/mips/pmap.h
include/mips/pmap.h
@@ -28,7 +28,7 @@ typedef uint32_t pde_t;
#define PD_ENTRIES (PAGESIZE / (int)sizeof(pde_t))
static_assert(PD_ENTRIES == 1 << 10,
- "Page table describes 10-bit range addresses!");
+ "Page directory describes 10-bit range addresses!");
/* Number of page table entries. */
#define PT_ENTRIES (PAGESIZE / (int)sizeof(pte_t))
@@ -47,10 +47,6 @@ static_assert(PT_ENTRIES == 1 << 10,
#define PMAP_USER_BEGIN 0x00001000
#define PMAP_USER_END 0x80000000
-#ifndef __ASSEMBLER__
-void *pmap_kseg2_to_kseg0(void *va);
-#endif /* __ASSEMBLER__ */
-
#endif /* !_MACHDEP */
#endif /* !_MIPS_PMAP_H_ */

MODIFY
include/mips/types.h
include/mips/types.h
#ifndef _MIPS_TYPES_H_
#define _MIPS_TYPES_H_
+#include <stdint.h>
+
typedef int32_t register_t;
typedef int32_t fpregister_t;

MODIFY
include/sys/pcpu.h
include/sys/pcpu.h
#ifndef _SYS_PCPU_H_
#define _SYS_PCPU_H_
+#include <machine/types.h>
+#include <machine/pcpu.h>
#include <stdbool.h>
typedef struct thread thread_t;
@@ -14,7 +16,9 @@ typedef struct pcpu {
thread_t *idle_thread; /*!< idle thread executed on this CPU */
pmap_t *curpmap; /*!< current page table */
vm_map_t *uspace; /*!< user space virtual memory map */
- void *ksp; /*!< (MIPS) sp restored on user->kernel transition */
+
+ /* Machine-dependent part */
+ PCPU_MD_FIELDS;
} pcpu_t;
extern pcpu_t _pcpu_data[1];

MODIFY
sys/mips/boot.c
sys/mips/boot.c
__boot_data void *_kernel_end_kseg0;
/* Kernel page directory entries allocated in kseg0. */
__boot_data pde_t *_kernel_pmap_pde;
+/* The boot stack is used before we switch out to thread0. */
+static alignas(PAGESIZE) uint8_t _boot_stack[PAGESIZE];
/* Allocates pages in kseg0. The argument must be multiple of PAGESIZE. */
static __boot_text void *bootmem_alloc(size_t bytes) {
@@ -23,7 +25,7 @@ __boot_text static void halt(void) {
continue;
}
-__boot_text void mips_init(void) {
+__boot_text void *mips_init(void) {
/*
* Ensure we're in kernel mode, disable FPU,
* leave error level & exception level and disable interrupts.
@@ -107,6 +109,12 @@ __boot_text void mips_init(void) {
mips32_setentrylo1(PTE_PFN(MIPS_KSEG0_TO_PHYS(pde)) | PTE_KERNEL);
mips32_setindex(0);
mips32_tlbwi();
+
+ /* Return the end of boot stack (grows downwards on MIPS) as new sp.
+ * This is done in order to move kernel boot process to kseg2, since
+ * current KASAN implementation requires all instrumented stack accesses
+ * to be done through kseg2. */
+ return &_boot_stack[PAGESIZE];
}
/* Following code is used by gdb scripts. */

MODIFY
sys/mips/context.c
sys/mips/context.c
@@ -7,7 +7,7 @@ void ctx_init(ctx_t *ctx, void *pc, void *sp) {
bzero(ctx, sizeof(ctx_t));
ctx->pc = (register_t)pc;
- ctx->sp = (register_t)pmap_kseg2_to_kseg0(sp);
+ ctx->sp = (register_t)sp;
/* Take SR from caller's context and enable interrupts. */
ctx->sr = (register_t)mips32_get_c0(C0_STATUS) | SR_IE;
}

MODIFY
sys/mips/ebase.S
sys/mips/ebase.S
#define LOAD_FPU_REG(reg, offset, base) lwc1 reg, (EXC_FPU_##offset)(base)
-#define SAVE_CPU_CTX(_sp, reg) \
+#define SAVE_CPU_CTX(reg) \
SAVE_REG_CFI(AT, AT, reg); \
SAVE_REG_CFI(v0, V0, reg); \
SAVE_REG_CFI(v1, V1, reg); \
SAVE_REG_CFI(t8, T8, reg); \
SAVE_REG_CFI(t9, T9, reg); \
SAVE_REG_CFI(gp, GP, reg); \
- SAVE_REG(_sp, SP, reg); \
- .cfi_rel_offset sp, EXC_SP; \
SAVE_REG_CFI(fp, FP, reg); \
/* Save value of user-space RA just before syscall. */ \
SAVE_REG(ra, RA, reg); \
mflo t0; \
mfhi t1; \
SAVE_REG(t0, LO, reg); \
- SAVE_REG(t1, HI, reg); \
- mfc0 t0, C0_STATUS; \
- mfc0 t1, C0_CAUSE; \
- mfc0 t2, C0_BADVADDR; \
- mfc0 t3, C0_EPC; \
- SAVE_REG(t0, SR, reg); \
- SAVE_REG(t1, CAUSE, reg); \
- SAVE_REG(t2, BADVADDR, reg); \
- /* Save value of user-space PC just before syscall. */ \
- SAVE_REG(t3, PC, reg); \
- .cfi_return_column t3; \
- .cfi_rel_offset t3, EXC_PC
+ SAVE_REG(t1, HI, reg);
#define LOAD_CPU_CTX() \
- LOAD_REG(t0, PC, sp); \
- mtc0 t0, C0_EPC; \
LOAD_REG(t0, HI, sp); \
LOAD_REG(t1, LO, sp); \
mthi t0; \
LOAD_FPU_REG($f31, F31, sp); \
LOAD_FPU_REG($31, FSR, sp)
+#define LOAD_PCPU_KSEG0(reg) \
+ LA reg, MIPS_KSEG2_TO_KSEG0(_pcpu_data)
+
# Increment interrupt disable nest level.
#define TD_INC_IDNEST(reg) \
lw t0, TD_IDNEST(reg); \
ehb; \
ext reg, reg, SR_IMASK_SHIFT, SR_IMASK_BITS
+# Remove given bits from the status register.
+#define STATUS_CLEAR(mask) \
+ mfc0 k0, C0_STATUS; \
+ li k1, ~(mask); \
+ and k0, k1; \
+ mtc0 k0, C0_STATUS
+
+#define PCPU_SAVE(reg) \
+ mfc0 k1, C0_##reg; \
+ sw k1, PCPU_##reg(k0)
+
+#define PCPU_RESTORE(reg) \
+ lw k1, PCPU_##reg(k0); \
+ mtc0 k1, C0_##reg
+
+#define PCPU_TO_FRAME(reg) \
+ lw t1, PCPU_##reg(t0); \
+ SAVE_REG(t1, reg, sp)
.global _ebase
@@ -289,28 +295,49 @@ exc_enter:
beqz k0, kern_exc_enter
nop
-# IMPORTANT: While exception level is active the code may only access kernel
-# stack and local pcpu structure! FTTB These are accessed through kseg0.
-# It is UNSAFE to use k0/k1 registers when TLB miss handler may be triggered!
+# IMPORTANT:
+# - While exception level is active, the code may only access local pcpu
+# structure! FTTB It's accessed through kseg0.
+# - It is UNSAFE to use k0/k1 registers when TLB miss handler may be triggered!
+# - Status, cause, epc & badvaddr registers cannot be saved directly to kernel
+# stack (kseg2) as it may trigger TLB handler which would overwrite them!
SNESTED(user_exc_enter, USER_EXC_FRAME_SIZE, ra)
.cfi_signal_frame
- # Move user sp to temporary register and replace it with kernel sp.
- move k1, sp
- # Previous value of sp is saved in register k1.
- .cfi_register sp, k1
-
- # Exception frame will be saved on kernel stack.
- # Fetch exception frame pointer where context will be saved.
+ # Fetch PCPU address
LOAD_PCPU_KSEG0(k0)
- # sp contains kernel stack pointer.
+ # Store sp, status, cause, epc, badvaddr registers in PCPU
+ sw sp, PCPU_SP(k0)
+ PCPU_SAVE(STATUS)
+ PCPU_SAVE(CAUSE)
+ PCPU_SAVE(EPC)
+ PCPU_SAVE(BADVADDR)
+
+ # sp now contains kernel stack pointer.
lw sp, PCPU_KSP(k0)
- # Define a rule for computing CFA as: *sp + 0.
+ # Define a rule for computing CFA as: sp + 0
.cfi_def_cfa sp, 0
- SAVE_CPU_CTX(k1, sp)
+
+ # We cannot access stack (KSEG2) when exception level is enabled
+ # Let's enter kernel mode, drop exception level, disable interrupts
+ STATUS_CLEAR(SR_KSU_MASK|SR_EXL|SR_IE)
+
+ # Save context on the kernel stack
+ SAVE_CPU_CTX(sp)
+
+ # Copy sp, status, cause, epc, badvaddr from PCPU to stack frame
+ LOAD_PCPU_KSEG0(t0)
+ PCPU_TO_FRAME(SP)
+ .cfi_rel_offset sp, EXC_SP
+ PCPU_TO_FRAME(SR)
+ PCPU_TO_FRAME(CAUSE)
+ PCPU_TO_FRAME(PC)
+ .cfi_return_column t1
+ .cfi_rel_offset t1, EXC_PC
+ PCPU_TO_FRAME(BADVADDR)
# If FPU is enabled save FPU registers.
mfc0 t0, C0_STATUS
@@ -324,12 +351,8 @@ skip_fpu_save:
# Load kernel global pointer.
la gp, _gp
- # Turn off FPU, enter kernel mode,
- # drop exception level and disable interrupts.
- mfc0 t0, C0_STATUS
- li t1, ~(SR_CU1|SR_KSU_MASK|SR_EXL|SR_IE)
- and t0, t1
- mtc0 t0, C0_STATUS
+ # Turn off FPU
+ STATUS_CLEAR(SR_CU1)
# Fetch thread control block pointer to s0 for later use.
LOAD_PCPU(s0)
@@ -352,21 +375,23 @@ user_exc_leave:
# Set current stack pointer to user exception frame.
# This is crucial on first entry to user-space for this thread.
- LOAD_PCPU(s0)
+ LOAD_PCPU_KSEG0(s0)
lw t0, PCPU_CURTHREAD(s0)
lw sp, TD_UFRAME(t0)
# Update kernel stack pointer to be used on kernel reentry.
sw sp, PCPU_KSP(s0)
- # Update status register held in exception frame (only interrupt mask).
+ # Load status register from exception frame, update it with current
+ # interrupt mask and exception mode, and store it within PCPU
LOAD_REG(t0, SR, sp)
ins t0, t1, SR_IMASK_SHIFT, SR_IMASK_BITS
- SAVE_REG(t0, SR, sp)
-
- # Enter exception level with user-mode settings.
ori t0, SR_EXL
- mtc0 t0, C0_STATUS
+ sw t0, PCPU_SR(s0)
+
+ # Copy pc register from stack frame to PCPU
+ LOAD_REG(t0, PC, sp)
+ sw t0, PCPU_PC(s0)
# If FPU has been enabled, then restore FPU registers.
ext t1, t0, SR_CU1_SHIFT, 1
@@ -379,6 +404,11 @@ skip_fpu_restore:
# Load context from exception frame on stack, sp will get overwritten.
LOAD_CPU_CTX()
+ # Restore status and epc registers from PCPU
+ LOAD_PCPU_KSEG0(k0)
+ PCPU_RESTORE(STATUS)
+ PCPU_RESTORE(EPC)
+
# This label is useful for debugging.
sync
user_return:
@@ -386,10 +416,33 @@ user_return:
END(user_exc_enter)
SNESTED(kern_exc_enter, KERN_EXC_FRAME_SIZE, ra)
+ # Fetch PCPU address
+ LOAD_PCPU_KSEG0(k0)
+
+ # Store sp, status, cause, epc, badvaddr registers in PCPU
+ sw sp, PCPU_SP(k0)
+ PCPU_SAVE(STATUS)
+ PCPU_SAVE(CAUSE)
+ PCPU_SAVE(EPC)
+ PCPU_SAVE(BADVADDR)
+
# Allocate stack frame (of KERN_EXC_FRAME_SIZE bytes) and save context.
- move k0, sp # save original sp value
subu sp, KERN_EXC_FRAME_SIZE # required by gdb to recognize frame
- SAVE_CPU_CTX(k0, sp)
+
+ # We cannot access stack (KSEG2) when exception level is enabled
+ # Let's drop exception level and disable interrupts.
+ STATUS_CLEAR(SR_EXL|SR_IE)
+
+ # Save context on the kernel stack
+ SAVE_CPU_CTX(sp)
+
+ # Copy sp, status, cause, epc, badvaddr from PCPU to stack frame
+ LOAD_PCPU_KSEG0(t0)
+ PCPU_TO_FRAME(SP)
+ PCPU_TO_FRAME(SR)
+ PCPU_TO_FRAME(CAUSE)
+ PCPU_TO_FRAME(PC)
+ PCPU_TO_FRAME(BADVADDR)
# In normal conditions we're going to enter regular exception handler.
la s1, mips_exc_handler
@@ -407,12 +460,6 @@ SNESTED(kern_exc_enter, KERN_EXC_FRAME_SIZE, ra)
# Load kernel global pointer.
1: la gp, _gp
- # Drop exception level and disable interrupts.
- mfc0 t0, C0_STATUS
- li t1, ~(SR_EXL|SR_IE)
- and t0, t1
- mtc0 t0, C0_STATUS
-
# Fetch thread control block pointer to s0 for later use.
LOAD_PCPU(t0)
lw s0, PCPU_CURTHREAD(t0)
@@ -431,18 +478,27 @@ SNESTED(kern_exc_enter, KERN_EXC_FRAME_SIZE, ra)
kern_exc_leave:
DISABLE_INT(t1)
- # Load status register from exception frame and update it with current
- # interrupt mask.
+ LOAD_PCPU_KSEG0(k0)
+
+ # Load status register from exception frame, update it with current
+ # interrupt mask and exception mode, and store it within PCPU
LOAD_REG(t0, SR, sp)
ins t0, t1, SR_IMASK_SHIFT, SR_IMASK_BITS
-
- # Enter exception level with kernel-mode settings.
ori t0, SR_EXL
- mtc0 t0, C0_STATUS
+ sw t0, PCPU_SR(k0)
+
+ # Copy pc register from exception frame to PCPU
+ LOAD_REG(t0, PC, sp)
+ sw t0, PCPU_PC(k0)
# Load context from exception frame on stack, sp will get overwritten.
LOAD_CPU_CTX()
+ # Restore status and epc registers from PCPU
+ LOAD_PCPU_KSEG0(k0)
+ PCPU_RESTORE(STATUS)
+ PCPU_RESTORE(EPC)
+
sync
eret
END(kern_exc_enter)

MODIFY
sys/mips/exception.c
sys/mips/exception.c
@@ -34,7 +34,7 @@ void exc_frame_init(exc_frame_t *frame, void *pc, void *sp, unsigned flags) {
frame->gp = usermode ? 0 : (register_t)_gp;
frame->pc = (register_t)pc;
- frame->sp = (register_t)pmap_kseg2_to_kseg0(sp);
+ frame->sp = (register_t)sp;
/* For user-mode exception frame we must make sure that:
* - user mode is active,

MODIFY
sys/mips/genassym.cf
sys/mips/genassym.cf
@@ -113,5 +113,13 @@ define CTX_FRAME_SIZE sizeof(ctx_t)
define P_USPACE offsetof(proc_t, p_uspace)
-define PCPU_CURTHREAD offsetof(pcpu_t, curthread)
+# PC/EPC and SR/STATUS definitions are duplicated for convenience reasons
+define PCPU_SP offsetof(pcpu_t, sp)
+define PCPU_PC offsetof(pcpu_t, epc)
+define PCPU_EPC offsetof(pcpu_t, epc)
define PCPU_KSP offsetof(pcpu_t, ksp)
+define PCPU_CAUSE offsetof(pcpu_t, cause)
+define PCPU_SR offsetof(pcpu_t, status)
+define PCPU_STATUS offsetof(pcpu_t, status)
+define PCPU_BADVADDR offsetof(pcpu_t, badvaddr)
+define PCPU_CURTHREAD offsetof(pcpu_t, curthread)

MODIFY
sys/mips/malta.c
sys/mips/malta.c
@@ -118,7 +118,7 @@ static void *malta_kenv(int argc, char **argv, char **envp) {
kenv_bootstrap(kenvp, kinit);
- return (void *)MIPS_KSEG2_TO_KSEG0(stk->stk_ptr);
+ return stk->stk_ptr;
}
intptr_t ramdisk_get_start(void) {

MODIFY
sys/mips/pmap.c
sys/mips/pmap.c
@@ -295,14 +295,6 @@ void pmap_copy_page(vm_page_t *src, vm_page_t *dst) {
memcpy(PG_KSEG0_ADDR(dst), PG_KSEG0_ADDR(src), PAGESIZE);
}
-void *pmap_kseg2_to_kseg0(void *va) {
- if (!MIPS_IN_KSEG2_P(va))
- return va;
- paddr_t pa;
- assert(pmap_extract(pmap_kernel(), (vaddr_t)va, &pa));
- return (void *)MIPS_PHYS_TO_KSEG0(pa);
-}
-
/* TODO: at any given moment there're two page tables in use:
* - kernel-space pmap for kseg2 & kseg3
* - user-space pmap for useg

MODIFY
sys/mips/start.S
sys/mips/start.S
#include <mips/asm.h>
#include <mips/pcpu.h>
#include <mips/regdef.h>
-#include <mips/m32c0.h>
#include "assym.h"
@@ -32,6 +31,7 @@ LEAF(_start)
move s3, a3
jal mips_init
+ move sp, v0 # Set boot stack (already in kseg2)
# From this point kernel sections are accessible through kseg2!
move a0, s0

MODIFY
sys/mips/thread.c
sys/mips/thread.c
@@ -19,7 +19,7 @@ void thread_entry_setup(thread_t *td, entry_fn_t target, void *arg) {
exc_frame_t *kframe = kstack_alloc_s(stk, cpu_exc_frame_t);
ctx_t *kctx = kstack_alloc_s(stk, ctx_t);
- td->td_uframe = pmap_kseg2_to_kseg0(uframe);
+ td->td_uframe = uframe;
td->td_kframe = kframe;
td->td_kctx = kctx;

Access kernel stack using only KSEG2 addresses (#636)