Skip to content

runtime: move goroutine state into stack-rooted getg context #2074

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cpunion wants to merge 10 commits into from
Closed

runtime: move goroutine state into stack-rooted getg context #2074

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
b6e408d
compiler: add standalone locality directive analysis
cpunion Jul 13, 2026
9cd92d4
runtime: add stack-rooted locality contexts
cpunion Jul 14, 2026
0a1e742
compiler: lower TLS and GLS package variables
cpunion Jul 14, 2026
364a246
build: key local context mode in package cache
cpunion Jul 14, 2026
c636e95
test: cover TLS and GLS runtime behavior
cpunion Jul 14, 2026
334c8a0
runtime: split local package fast path
cpunion Jul 14, 2026
c3379be
runtime: return local package data from hot block
cpunion Jul 14, 2026
7f7ce3b
test: cover locality debug metadata lowering
cpunion Jul 14, 2026
7b46291
runtime: consolidate goroutine state behind getg
cpunion Jul 13, 2026
89701f5
test: cover runtime g state isolation
cpunion Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions cl/_testdata/cpkg/in.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,25 @@ package C

// CHECK-LABEL: define double @Double(double %0){{.*}} {
// CHECK-NEXT: _llgo_0:
// CHECK-NEXT: %1 = fmul double 2.000000e+00, %0
// CHECK-NEXT: ret double %1
// CHECK-NEXT: %1 = alloca %"{{.*}}LocalContext", align 8
// CHECK-NEXT: call void @llvm.memset.p0.i64(ptr %1, i8 0, i64 32, i1 false)
// CHECK-NEXT: %2 = call i64 @"{{.*}}EnterLocalContext"(ptr %1)
// CHECK-NEXT: %3 = fmul double 2.000000e+00, %0
// CHECK-NEXT: call void @"{{.*}}LeaveLocalContext"(ptr %1, i64 %2)
// CHECK-NEXT: ret double %3
// CHECK-NEXT: }
func Double(x float64) float64 {
return 2 * x
}

// CHECK-LABEL: define i64 @add(i64 %0, i64 %1){{.*}} {
// CHECK-NEXT: _llgo_0:
// CHECK-NEXT: %2 = call i64 @"{{.*}}.add"(i64 %0, i64 %1)
// CHECK-NEXT: ret i64 %2
// CHECK-NEXT: %2 = alloca %"{{.*}}LocalContext", align 8
// CHECK-NEXT: call void @llvm.memset.p0.i64(ptr %2, i8 0, i64 32, i1 false)
// CHECK-NEXT: %3 = call i64 @"{{.*}}EnterLocalContext"(ptr %2)
// CHECK-NEXT: %4 = call i64 @"{{.*}}.add"(i64 %0, i64 %1)
// CHECK-NEXT: call void @"{{.*}}LeaveLocalContext"(ptr %2, i64 %3)
// CHECK-NEXT: ret i64 %4
// CHECK-NEXT: }
func Xadd(a, b int) int {
return add(a, b)
Expand Down
26 changes: 17 additions & 9 deletions cl/_testgo/goroutine/in.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,20 +40,28 @@ func main() {

// CHECK-LABEL: define ptr @"{{.*}}goroutine._llgo_routine$1"(ptr %0){{.*}} {
// CHECK-NEXT: _llgo_0:
// CHECK-NEXT: %1 = load { %"{{.*}}String" }, ptr %0, align 8
// CHECK-NEXT: %2 = extractvalue { %"{{.*}}String" } %1, 0
// CHECK-NEXT: %1 = alloca %"{{.*}}LocalContext", align 8
// CHECK-NEXT: call void @llvm.memset.p0.i64(ptr %1, i8 0, i64 32, i1 false)
// CHECK-NEXT: %2 = call i64 @"{{.*}}EnterLocalContext"(ptr %1)
// CHECK-NEXT: %3 = load { %"{{.*}}String" }, ptr %0, align 8
// CHECK-NEXT: %4 = extractvalue { %"{{.*}}String" } %3, 0
// CHECK-NEXT: call void @"{{.*}}FreeRoot"(ptr %0)
// CHECK-NEXT: call void @"{{.*}}PrintString"(%"{{.*}}String" %2)
// CHECK-NEXT: call void @"{{.*}}PrintString"(%"{{.*}}String" %4)
// CHECK-NEXT: call void @"{{.*}}PrintByte"(i8 10)
// CHECK-NEXT: call void @"{{.*}}LeaveLocalContext"(ptr %1, i64 %2)
// CHECK-NEXT: ret ptr null

// CHECK-LABEL: define ptr @"{{.*}}goroutine._llgo_routine$2"(ptr %0){{.*}} {
// CHECK-NEXT: _llgo_0:
// CHECK-NEXT: %1 = load { { ptr, ptr }, %"{{.*}}String" }, ptr %0, align 8
// CHECK-NEXT: %2 = extractvalue { { ptr, ptr }, %"{{.*}}String" } %1, 0
// CHECK-NEXT: %3 = extractvalue { { ptr, ptr }, %"{{.*}}String" } %1, 1
// CHECK-NEXT: %1 = alloca %"{{.*}}LocalContext", align 8
// CHECK-NEXT: call void @llvm.memset.p0.i64(ptr %1, i8 0, i64 32, i1 false)
// CHECK-NEXT: %2 = call i64 @"{{.*}}EnterLocalContext"(ptr %1)
// CHECK-NEXT: %3 = load { { ptr, ptr }, %"{{.*}}String" }, ptr %0, align 8
// CHECK-NEXT: %4 = extractvalue { { ptr, ptr }, %"{{.*}}String" } %3, 0
// CHECK-NEXT: %5 = extractvalue { { ptr, ptr }, %"{{.*}}String" } %3, 1
// CHECK-NEXT: call void @"{{.*}}FreeRoot"(ptr %0)
// CHECK-NEXT: %4 = extractvalue { ptr, ptr } %2, 1
// CHECK-NEXT: %5 = extractvalue { ptr, ptr } %2, 0
// CHECK-NEXT: call void %5(ptr %4, %"{{.*}}String" %3)
// CHECK-NEXT: %6 = extractvalue { ptr, ptr } %4, 1
// CHECK-NEXT: %7 = extractvalue { ptr, ptr } %4, 0
// CHECK-NEXT: call void %7(ptr %6, %"{{.*}}String" %5)
// CHECK-NEXT: call void @"{{.*}}LeaveLocalContext"(ptr %1, i64 %2)
// CHECK-NEXT: ret ptr null
74 changes: 74 additions & 0 deletions cl/_testgo/localitycodegen/in.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// LITTEST
package main

// CHECK-DAG: @"{{.*}}localitycodegen.Scalar" = thread_local global i64 0
// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_local_key" = global i8 0
// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$guard" = thread_local global i8 0
// CHECK-DAG: @"{{.*}}localitycodegen.__llgo_tls_init$failure" = global i8 0
// CHECK-NOT: RegisterLocalRoot
// CHECK-NOT: localitycodegen.Pointer" = thread_local
// CHECK-NOT: localitycodegen.Initialized" = thread_local

// CHECK-LABEL: define ptr @"{{.*}}localitycodegen.__llgo_local_block"()
// CHECK: call ptr @"{{.*}}runtime.LocalPackage"(ptr @"{{.*}}localitycodegen.__llgo_local_key", i64 16, i64 8)
// CHECK: ret ptr

// CHECK-LABEL: define void @"{{.*}}localitycodegen.__llgo_tls_init"()
// CHECK: call void @"{{.*}}localitycodegen.__llgo_local_init_0"()

// CHECK-LABEL: define void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"()
// CHECK: load i8, ptr
// CHECK: call void @"{{.*}}runtime.EnsureLocalInitializer"(ptr @"{{.*}}localitycodegen.__llgo_tls_init$guard", ptr @"{{.*}}localitycodegen.__llgo_tls_init$failure"

// CHECK-LABEL: define ptr @{{"?ExportedLocality"?}}()
// CHECK: call i64 @"{{.*}}EnterLocalContext"
// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"()
// CHECK: call void @"{{.*}}LeaveLocalContext"
// CHECK: ret ptr

// CHECK-LABEL: define void @"{{.*}}localitycodegen.init"()
// CHECK: store i8 2, ptr
// CHECK: call ptr @"{{.*}}localitycodegen.newPointer"()
// CHECK: call void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"()
// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"()

// CHECK-LABEL: define { i64, ptr, ptr } @"{{.*}}localitycodegen.values"()
// CHECK: call void @"{{.*}}localitycodegen.__llgo_tls_init$ensure"()
// CHECK: load i64, ptr @"{{.*}}localitycodegen.Scalar"
// CHECK: call ptr @"{{.*}}localitycodegen.__llgo_local_block"()
// CHECK: load ptr, ptr
// CHECK: load ptr, ptr

// CHECK-LABEL: define ptr @"{{.*}}localitycodegen._llgo_routine$1"(ptr %0)
// CHECK: alloca %"{{.*}}LocalContext", align 8
// CHECK: call i64 @"{{.*}}EnterLocalContext"
// CHECK: call void @"{{.*}}LeaveLocalContext"

var backing int

func newPointer() *int {
return &backing
}

//llgo:tls
var Scalar int

//llgo:gls
var Pointer *int

//llgo:tls
var Initialized = newPointer()

func values() (int, *int, *int) {
return Scalar, Pointer, Initialized
}

//export ExportedLocality
func ExportedLocality() *int {
return Pointer
}

func main() {
_, _, _ = values()
go values()
}
14 changes: 9 additions & 5 deletions cl/_testgo/selects/in.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,11 +234,15 @@ func main() {

// CHECK-LABEL: define ptr @"{{.*}}/cl/_testgo/selects._llgo_routine$1"(ptr %0){{.*}} {
// CHECK-NEXT: _llgo_0:
// CHECK-NEXT: %1 = load { { ptr, ptr } }, ptr %0, align 8
// CHECK-NEXT: %2 = extractvalue { { ptr, ptr } } %1, 0
// CHECK-NEXT: %1 = alloca %"{{.*}}/runtime/internal/runtime.LocalContext", align 8
// CHECK-NEXT: call void @llvm.memset.p0.i64(ptr %1, i8 0, i64 32, i1 false)
// CHECK-NEXT: %2 = call i64 @"{{.*}}/runtime/internal/runtime.EnterLocalContext"(ptr %1)
// CHECK-NEXT: %3 = load { { ptr, ptr } }, ptr %0, align 8
// CHECK-NEXT: %4 = extractvalue { { ptr, ptr } } %3, 0
// CHECK-NEXT: call void @"{{.*}}/runtime/internal/runtime.FreeRoot"(ptr %0)
// CHECK-NEXT: %3 = extractvalue { ptr, ptr } %2, 1
// CHECK-NEXT: %4 = extractvalue { ptr, ptr } %2, 0
// CHECK-NEXT: call void %4(ptr %3)
// CHECK-NEXT: %5 = extractvalue { ptr, ptr } %4, 1
// CHECK-NEXT: %6 = extractvalue { ptr, ptr } %4, 0
// CHECK-NEXT: call void %6(ptr %5)
// CHECK-NEXT: call void @"{{.*}}/runtime/internal/runtime.LeaveLocalContext"(ptr %1, i64 %2)
// CHECK-NEXT: ret ptr null
// CHECK-NEXT: }
40 changes: 38 additions & 2 deletions cl/compile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ type context struct {

staticGlobalInits map[*ssa.Global]llssa.Expr
staticInitStores map[*ssa.Store]none
locality localityLowering
}

func (p *context) rewriteValue(name string) (string, bool) {
Expand Down Expand Up @@ -389,7 +390,10 @@ func (p *context) compileGlobal(pkg llssa.Package, gbl *ssa.Global) {
return
}
dbgInstrln("==> NewVar", name, typ)
g := pkg.NewVar(name, typ, llssa.Background(vtype))
g, skip := p.localityGlobalStorage(pkg, gbl, name, typ, llssa.Background(vtype))
if skip {
return
}
if p.tryEmbedGlobalInit(pkg, gbl, g, name) {
return
}
Expand Down Expand Up @@ -599,12 +603,15 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun
dbgSymsEnabled := enableDbgSyms && (f == nil || f.Origin() == nil)
p.inits = append(p.inits, func() {
oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark := p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark
oldLocalityFunction := p.locality.function
p.fn = fn
p.goFn = f
p.callerFrameMark = llssa.Nil
p.locality.function = localityFunction{}
p.state = state // restore pkgState when compiling funcBody
defer func() {
p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark = oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark
p.locality.function = oldLocalityFunction
}()
p.phis = nil
if dbgSymsEnabled {
Expand All @@ -620,6 +627,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun
bodyPos := p.getFuncBodyPos(f)
b.DebugFunction(fn, pos, bodyPos)
}
p.prepareExportedLocalContext(f)
p.bvals = make(map[ssa.Value]llssa.Expr)
p.methodNilDerefChecks = collectMethodNilDerefChecks(f)
off := make([]int, len(f.Blocks))
Expand Down Expand Up @@ -815,6 +823,9 @@ func (p *context) debugParams(b llssa.Builder, f *ssa.Function) {
}

func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, doModInit bool) llssa.BasicBlock {
oldLocalBlock := p.locality.function.block
p.locality.function.block = block
defer func() { p.locality.function.block = oldLocalBlock }()
var last int
var pyModInit bool
var prog = p.prog
Expand All @@ -823,6 +834,9 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do
var instrs = block.Instrs[n:]
var ret = fn.Block(block.Index)
b.SetBlock(ret)
if block.Index == 0 {
p.enterExportedLocalContext(b)
}
if block.Index == 0 && p.shouldTrackCallerFrames() {
p.pushCallerLocationFrame(b, block.Parent())
}
Expand All @@ -835,6 +849,7 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do
}

if doModInit {
p.initializeLocalGuards(b)
if p.state != pkgInPatch {
p.applyEmbedInits(b)
}
Expand Down Expand Up @@ -1600,6 +1615,7 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) {
if p.shouldTrackCallerFrames() {
p.popCallerLocationFrame(b)
}
p.leaveExportedLocalContext(b)
b.Return(results...)
case *ssa.If:
fn := p.fn
Expand Down Expand Up @@ -1694,7 +1710,7 @@ func (p *context) compileValue(b llssa.Builder, v ssa.Value) llssa.Expr {
if isCgoVar(varName) {
p.cgoSymbols = append(p.cgoSymbols, val.Name())
}
if enableDbgSyms {
if enableDbgSyms && p.localityAllowsGlobalDebug(v) {
pos := p.fset.Position(v.Pos())
b.DIGlobal(val, v.Name(), pos)
}
Expand Down Expand Up @@ -1930,6 +1946,15 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri
pkg.Pkg = pkgTypes
patch.Alt.Pkg = pkgTypes
}
if err = ParsePkgSyntax(prog, pkgProg.Fset, pkgTypes, files); err != nil {
return nil, nil, err
}
if err = prog.ValidateLocalities(llssa.PathOf(pkgTypes)); err != nil {
return nil, nil, err
}
if err = validateLocalInitializers(prog, pkgTypes); err != nil {
return nil, nil, err
}
if pkgPath == llssa.PkgRuntime {
prog.SetRuntime(pkgTypes)
}
Expand Down Expand Up @@ -2030,6 +2055,17 @@ func processPkg(ctx *context, ret llssa.Package, pkg *ssa.Package) {
sort.Slice(members, func(i, j int) bool {
return members[i].name < members[j].name
})
localGlobals := make([]*ssa.Global, 0)
for _, m := range members {
global, ok := m.val.(*ssa.Global)
if !ok || isCgoFuncPtrVar(global.Name()) {
continue
}
localGlobals = append(localGlobals, global)
}
// Address accessors and replay guards must exist before any function body
// can reference a local package variable, regardless of member sort order.
ctx.prepareLocalVariables(ret, localGlobals)

for _, m := range members {
member := m.val
Expand Down
Loading
Loading