Skip to content

Save all volatile argument registers in GC probe hijack frames on AMD64 and ARM64 #126848

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MichalStrehovsky merged 16 commits into from
Apr 17, 2026
+271 −139
Merged

Save all volatile argument registers in GC probe hijack frames on AMD64 and ARM64 #126848

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
4620ee6
Initial plan
Copilot Apr 13, 2026
b8e5153
Save all volatile argument registers (RDX, R8, R9) in GC probe hijack…
Copilot Apr 13, 2026
4184db5
Improve alloc_stack comment to describe what is being allocated
Copilot Apr 13, 2026
d671e50
Also save RSI and RDI in Unix AMD64 GC probe hijack frames
Copilot Apr 13, 2026
c80100d
Potential fix for pull request finding
jkotas Apr 14, 2026
87b210e
Use pop_register/pop_nonvol_reg/free_stack in POP_PROBE_FRAME for CFI…
Copilot Apr 14, 2026
a2a12bb
Save volatile argument registers x3-x7 in GC probe hijack frames on A…
Copilot Apr 14, 2026
b1486c3
Update RBM_INTERFACELOOKUP_FOR_SLOT_TRASH and save all FP argument re…
Copilot Apr 14, 2026
246d39d
Update FixupHijackedCallstack comments to include preserved FP registers
Copilot Apr 14, 2026
10598c1
Expand RBM_INTERFACELOOKUP_FOR_SLOT_TRASH to trash all non-argument r…
Copilot Apr 14, 2026
cd43a53
Cherry-pick JIT PHI jump threading (PR #126812) with targeted fix for…
Copilot Apr 16, 2026
5a82a7d
Revert "Cherry-pick JIT PHI jump threading (PR #126812) with targeted…
Copilot Apr 16, 2026
2b6ff21
Merge remote-tracking branch 'origin/main' into copilot/fix-fast-tail…
Copilot Apr 16, 2026
7442a6d
Merge main and apply targeted JIT PHI jump threading fix for #126976
Copilot Apr 16, 2026
90a23b2
Add project exclusions for arm64 Windows tests
MichalStrehovsky Apr 16, 2026
4369c76
Fix JITDUMP formatting in redundantbranchopts.cpp to satisfy jitforma…
Copilot Apr 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,9 @@ PTFF_SAVE_ALL_PRESERVED equ 000000F7h ;; NOTE: RBP is not included in this set
PTFF_SAVE_RSP equ 00008000h
PTFF_SAVE_RAX equ 00000100h ;; RAX is saved in hijack handler - in case it contains a GC ref
PTFF_SAVE_RCX equ 00000200h ;; RCX is saved in hijack handler - in case it contains a GC ref
PTFF_SAVE_RDX equ 00000400h ;; RDX is saved in hijack handler - in case it contains a GC ref
PTFF_SAVE_R8 equ 00000800h ;; R8 is saved in hijack handler - in case it contains a GC ref
PTFF_SAVE_R9 equ 00001000h ;; R9 is saved in hijack handler - in case it contains a GC ref
PTFF_SAVE_ALL_SCRATCH equ 00007F00h
PTFF_THREAD_HIJACK equ 00100000h ;; indicates that this is a frame for a hijacked call

Expand Down
40 changes: 26 additions & 14 deletions src/coreclr/nativeaot/Runtime/amd64/GcProbe.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,26 @@
#include <unixasmmacros.inc>

//
// See PUSH_COOP_PINVOKE_FRAME, this macro is very similar, but also saves RAX/RCX/RDX and accepts the register
// bitmask in R8
// See PUSH_COOP_PINVOKE_FRAME, this macro is very similar, but also saves volatile argument registers
// and accepts the register bitmask
//
// On entry:
// - BITMASK: bitmask describing pushes, may be volatile register or constant value
// - RAX: managed function return value, may be an object or byref
// - RCX, RDX, R8, R9: may contain objects or byrefs at the hijack point
// - preserved regs: need to stay preserved, may contain objects or byrefs
//
// INVARIANTS
// - The macro assumes it is called from a prolog, prior to a frame pointer being setup.
// - All preserved registers remain unchanged from their values in managed code.
//
.macro PUSH_PROBE_FRAME threadReg, trashReg, BITMASK
push_register r9 // save R9, it might contain an objectref
Comment thread
jkotas marked this conversation as resolved.
push_register r8 // save R8, it might contain an objectref
push_register rdx // save RDX, it might contain an objectref
push_register rcx // save RCX, it might contain an objectref (async continuation)
push_register rax // save RAX, it might contain an objectref
lea \trashReg, [rsp + 0x20]
lea \trashReg, [rsp + 0x30]
push_register \trashReg // save caller`s RSP
push_nonvol_reg r15 // save preserved registers
push_nonvol_reg r14 // ..
Expand All @@ -32,11 +35,11 @@
push_register \BITMASK // save the register bitmask passed in by caller
push_register \threadReg // Thread * (unused by stackwalker)
push_nonvol_reg rbp // save caller`s RBP
mov \trashReg, [rsp + 12*8] // Find the return address
mov \trashReg, [rsp + 14*8] // Find the return address
push_register \trashReg // save m_RIP
lea \trashReg, [rsp + 0] // trashReg == address of frame

// allocate space for xmm0, xmm1 and alignment
// allocate space for xmm0, xmm1
alloc_stack 0x20 + 0

// save xmm0 and xmm1 in case they are used as return values
Expand Down Expand Up @@ -67,6 +70,8 @@
pop rax
pop rcx
pop rdx
pop r8
pop r9
.endm

//
Expand All @@ -78,16 +83,20 @@
//
// Register state on exit:
// R11: thread pointer
// RAX, RCX, RDX preserved, other volatile regs trashed
// RAX, RCX, RDX, R8, R9 preserved, R10 trashed
//
.macro FixupHijackedCallstack
// preserve RAX, RDX as they may contain return values
// preserve volatile argument registers across INLINE_GETTHREAD
push rax
push rdx

// preserve RCX as it may contain async continuation return value
push rcx

// preserve R8 and R9 as they may contain GC refs
push r8
push r9

// align stack
sub rsp, 0x8

Expand All @@ -97,19 +106,22 @@

add rsp, 0x8

pop r9
pop r8

pop rcx

pop rdx
pop rax

// Fix the stack by pushing the original return address
mov r8, [r11 + OFFSETOF__Thread__m_pvHijackedReturnAddress]
push r8
mov r10, [r11 + OFFSETOF__Thread__m_pvHijackedReturnAddress]
push r10

// Clear hijack state
xor r8, r8
mov [r11 + OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation], r8
mov [r11 + OFFSETOF__Thread__m_pvHijackedReturnAddress], r8
xor r10, r10
mov [r11 + OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation], r10
mov [r11 + OFFSETOF__Thread__m_pvHijackedReturnAddress], r10
.endm

//
Expand All @@ -124,12 +136,12 @@ NESTED_ENTRY RhpGcProbeHijack, _TEXT, NoHandler
ret

LOCAL_LABEL(WaitForGC):
mov r8d, DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_RAX + PTFF_SAVE_RCX + PTFF_SAVE_RDX + PTFF_THREAD_HIJACK
mov r10d, DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_RAX + PTFF_SAVE_RCX + PTFF_SAVE_RDX + PTFF_SAVE_R8 + PTFF_SAVE_R9 + PTFF_THREAD_HIJACK
jmp C_FUNC(RhpWaitForGC)
NESTED_END RhpGcProbeHijack, _TEXT

NESTED_ENTRY RhpWaitForGC, _TEXT, NoHandler
PUSH_PROBE_FRAME r11, rax, r8
PUSH_PROBE_FRAME r11, rax, r10
END_PROLOGUE

mov rbx, r11
Expand Down
55 changes: 31 additions & 24 deletions src/coreclr/nativeaot/Runtime/amd64/GcProbe.asm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@
include AsmMacros.inc

;;
;; See PUSH_COOP_PINVOKE_FRAME, this macro is very similar, but also saves RAX/RCX and accepts
;; the register bitmask
;; See PUSH_COOP_PINVOKE_FRAME, this macro is very similar, but also saves volatile argument registers
;; and accepts the register bitmask
;;
;; On entry:
;; - BITMASK: bitmask describing pushes, a volatile register
;; - RAX: managed function return value, may be an object or byref
;; - RCX: managed function return value (async continuation), may be an object
;; - RDX, R8, R9: may contain objects or byrefs at the hijack point
;; - preserved regs: need to stay preserved, may contain objects or byrefs
;;
;; INVARIANTS
Expand All @@ -19,9 +20,12 @@ include AsmMacros.inc
;;
PUSH_PROBE_FRAME macro threadReg, trashReg, BITMASK

push_vol_reg r9 ; save R9, it might contain an objectref
push_vol_reg r8 ; save R8, it might contain an objectref
push_vol_reg rdx ; save RDX, it might contain an objectref
push_vol_reg rcx ; save RCX, it might contain an objectref (async continuation)
push_vol_reg rax ; save RAX, it might contain an objectref
lea trashReg, [rsp + 18h]
lea trashReg, [rsp + 30h]
push_vol_reg trashReg ; save caller's RSP
push_nonvol_reg r15 ; save preserved registers
push_nonvol_reg r14 ; ..
Expand All @@ -33,12 +37,12 @@ PUSH_PROBE_FRAME macro threadReg, trashReg, BITMASK
push_vol_reg BITMASK ; save the register bitmask passed in by caller
push_vol_reg threadReg ; Thread * (unused by stackwalker)
push_nonvol_reg rbp ; save caller's RBP
mov trashReg, [rsp + 13*8] ; Find the return address
mov trashReg, [rsp + 16*8] ; Find the return address
push_vol_reg trashReg ; save m_RIP
lea trashReg, [rsp + 0] ; trashReg == address of frame

;; allocate scratch space and any required alignment
alloc_stack 20h + 10h + 8
;; allocate scratch space (20h home space + 10h for xmm0)
alloc_stack 20h + 10h

;; save xmm0 in case it's being used as a return value
movdqa [rsp + 20h], xmm0
Expand All @@ -54,7 +58,7 @@ endm
;;
POP_PROBE_FRAME macro
movdqa xmm0, [rsp + 20h]
add rsp, 20h + 10h + 8 + 8 ; deallocate stack and discard saved m_RIP
add rsp, 20h + 10h + 8 ; deallocate scratch space and discard saved m_RIP
pop rbp
pop rax ; discard Thread*
pop rax ; discard BITMASK
Expand All @@ -68,6 +72,9 @@ POP_PROBE_FRAME macro
pop rax ; discard caller RSP
pop rax
pop rcx
pop rdx
pop r8
pop r9
endm

;;
Expand All @@ -78,21 +85,21 @@ endm
;; All registers correct for return to the original return address.
;;
;; Register state on exit:
;; RDX: thread pointer
;; RAX/RCX: preserved, other volatile regs trashed
;; R10: thread pointer
;; RAX/RCX/RDX/R8/R9: preserved, R11 trashed
;;
FixupHijackedCallstack macro
;; rdx <- GetThread(), TRASHES r8
INLINE_GETTHREAD rdx, r8
;; r10 <- GetThread(), TRASHES r11
INLINE_GETTHREAD r10, r11

;; Fix the stack by pushing the original return address
mov r8, [rdx + OFFSETOF__Thread__m_pvHijackedReturnAddress]
push r8
mov r11, [r10 + OFFSETOF__Thread__m_pvHijackedReturnAddress]
push r11

;; Clear hijack state
xor r8, r8
mov [rdx + OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation], r8
mov [rdx + OFFSETOF__Thread__m_pvHijackedReturnAddress], r8
xor r11, r11
mov [r10 + OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation], r11
mov [r10 + OFFSETOF__Thread__m_pvHijackedReturnAddress], r11
endm

;;
Expand All @@ -106,15 +113,15 @@ NESTED_ENTRY RhpGcProbeHijack, _TEXT
jnz @f
ret
@@:
mov r8d, DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_RAX + PTFF_SAVE_RCX + PTFF_THREAD_HIJACK
mov r11d, DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_RAX + PTFF_SAVE_RCX + PTFF_SAVE_RDX + PTFF_SAVE_R8 + PTFF_SAVE_R9 + PTFF_THREAD_HIJACK
jmp RhpWaitForGC
NESTED_END RhpGcProbeHijack, _TEXT

NESTED_ENTRY RhpWaitForGC, _TEXT
PUSH_PROBE_FRAME rdx, rax, r8
PUSH_PROBE_FRAME r10, rax, r11
END_PROLOGUE

mov rbx, rdx
mov rbx, r10
mov rcx, [rbx + OFFSETOF__Thread__m_pDeferredTransitionFrame]
call RhpWaitForGC2

Expand Down Expand Up @@ -147,7 +154,7 @@ ifdef FEATURE_GC_STRESS
;;
LEAF_ENTRY RhpGcStressHijack, _TEXT
FixupHijackedCallstack
mov r8d, DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_RAX + PTFF_SAVE_RCX
mov r11d, DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_RAX + PTFF_SAVE_RCX + PTFF_SAVE_RDX + PTFF_SAVE_R8 + PTFF_SAVE_R9
jmp RhpGcStressProbe
LEAF_END RhpGcStressHijack, _TEXT

Expand All @@ -157,15 +164,15 @@ LEAF_END RhpGcStressHijack, _TEXT
;; This worker performs the GC Stress work and returns to the original return address.
;;
;; Register state on entry:
;; RDX: thread pointer
;; R8: register bitmask
;; R10: thread pointer
;; R11: register bitmask
;;
;; Register state on exit:
;; Scratch registers, except for RAX/RCX, have been trashed
;; Scratch registers, except for RAX/RCX/RDX/R8/R9, have been trashed
;; All other registers restored as they were when the hijack was first reached.
;;
NESTED_ENTRY RhpGcStressProbe, _TEXT
PUSH_PROBE_FRAME rdx, rax, r8
PUSH_PROBE_FRAME r10, rax, r11
END_PROLOGUE

call RhpStressGc
Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/nativeaot/Runtime/unix/unixasmmacrosamd64.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,8 @@ C_FUNC(\Name):
#define PTFF_SAVE_RAX 0x00000100 // RAX is saved in hijack handler - in case it contains a GC ref
#define PTFF_SAVE_RCX 0x00000200 // RCX is saved in hijack handler - in case it contains a GC ref
#define PTFF_SAVE_RDX 0x00000400 // RDX is saved in hijack handler - in case it contains a GC ref
#define PTFF_SAVE_R8 0x00000800 // R8 is saved in hijack handler - in case it contains a GC ref
#define PTFF_SAVE_R9 0x00001000 // R9 is saved in hijack handler - in case it contains a GC ref
#define PTFF_SAVE_ALL_SCRATCH 0x00007F00
#define PTFF_THREAD_HIJACK 0x00100000 // indicates that this is a frame for a hijacked call

Expand Down
Loading