Skip to content

Commit 6dae01b

Browse files
authored
Add separate GC heap tunables (#13080)
* Add separate GC heap tunables Before this commit, the GC heap used a memory with identical tunables as regular Wasm linear memories. After this commit, the GC heap now uses a memory with its own tunables (`gc_heap_reservation`, `gc_heap_guard_size`, `gc_heap_reservation_for_growth`, and `gc_heap_may_move`) so that GC heaps can be configured independently from linear memories. These new GC heap tunables are also exposed as CLI flags and `Config` methods. We also generate pseudorandom values for these tunables in our fuzzing config generator. The pooling allocator uses the same pool for memories and GC heaps, so we validate that if you are using GC and the pooling allocator, then the new tunables must match the memory tunables. * Fix pooling allocator + GC heap tests * Addressing review feedback * cargo fmt
1 parent 1cc2930 commit 6dae01b

File tree

27 files changed

+1043
-126
lines changed

27 files changed

+1043
-126
lines changed

crates/cli-flags/src/lib.rs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,19 @@ wasmtime_option_group! {
6767
/// Size, in bytes, of guard pages for linear memories.
6868
pub memory_guard_size: Option<u64>,
6969

70+
/// Do not allow the GC heap to move in the host process's address
71+
/// space.
72+
pub gc_heap_may_move: Option<bool>,
73+
74+
/// Initial virtual memory allocation size for the GC heap.
75+
pub gc_heap_reservation: Option<u64>,
76+
77+
/// Bytes to reserve at the end of the GC heap for growth into.
78+
pub gc_heap_reservation_for_growth: Option<u64>,
79+
80+
/// Size, in bytes, of guard pages for the GC heap.
81+
pub gc_heap_guard_size: Option<u64>,
82+
7083
/// Indicates whether an unmapped region of memory is placed before all
7184
/// linear memories.
7285
pub guard_before_linear_memory: Option<bool>,
@@ -891,6 +904,19 @@ impl CommonOptions {
891904
if let Some(enable) = self.opts.guard_before_linear_memory {
892905
config.guard_before_linear_memory(enable);
893906
}
907+
908+
if let Some(size) = self.opts.gc_heap_reservation {
909+
config.gc_heap_reservation(size);
910+
}
911+
if let Some(enable) = self.opts.gc_heap_may_move {
912+
config.gc_heap_may_move(enable);
913+
}
914+
if let Some(size) = self.opts.gc_heap_guard_size {
915+
config.gc_heap_guard_size(size);
916+
}
917+
if let Some(size) = self.opts.gc_heap_reservation_for_growth {
918+
config.gc_heap_reservation_for_growth(size);
919+
}
894920
if let Some(enable) = self.opts.table_lazy_init {
895921
config.table_lazy_init(enable);
896922
}

crates/cranelift/src/bounds_checks.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -184,16 +184,17 @@ fn bounds_check_field_access(
184184
env.heap_access_spectre_mitigation() && clif_memory_traps_enabled;
185185

186186
let host_page_size_log2 = env.target_config().page_size_align_log2;
187+
let memory_tunables = heap.memory_tunables(env.tunables());
187188
let can_use_virtual_memory = heap
188189
.memory
189-
.can_use_virtual_memory(env.tunables(), host_page_size_log2)
190+
.can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2)
190191
&& clif_memory_traps_enabled;
191192
let can_elide_bounds_check = heap
192193
.memory
193-
.can_elide_bounds_check(env.tunables(), host_page_size_log2)
194+
.can_elide_bounds_check(&memory_tunables, host_page_size_log2)
194195
&& clif_memory_traps_enabled;
195-
let memory_guard_size = env.tunables().memory_guard_size;
196-
let memory_reservation = env.tunables().memory_reservation;
196+
let memory_guard_size = memory_tunables.guard_size();
197+
let memory_reservation = memory_tunables.reservation();
197198

198199
let offset_and_size = offset_plus_size(offset, access_size);
199200
let statically_in_bounds = statically_in_bounds(&builder.func, heap, index, offset_and_size);
@@ -348,7 +349,7 @@ fn bounds_check_field_access(
348349
// factor in the guard pages here.
349350
if can_use_virtual_memory
350351
&& heap.memory.minimum_byte_size().unwrap_or(u64::MAX) <= memory_reservation
351-
&& !heap.memory.memory_may_move(env.tunables())
352+
&& !heap.memory.memory_may_move(&memory_tunables)
352353
&& memory_reservation >= offset_and_size
353354
{
354355
let adjusted_bound = memory_reservation.checked_sub(offset_and_size).unwrap();

crates/cranelift/src/func_environ.rs

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ pub(crate) mod stack_switching;
44
use crate::BuiltinFunctionSignatures;
55
use crate::compiler::Compiler;
66
use crate::translate::{
7-
FuncTranslationStacks, GlobalVariable, Heap, HeapData, StructFieldsVec, TableData, TableSize,
8-
TargetEnvironment,
7+
FuncTranslationStacks, GlobalVariable, Heap, HeapData, MemoryKind, StructFieldsVec, TableData,
8+
TableSize, TargetEnvironment,
99
};
1010
use crate::trap::TranslateTrap;
1111
use cranelift_codegen::cursor::FuncCursor;
@@ -28,10 +28,10 @@ use wasmtime_core::math::f64_cvt_to_int_bounds;
2828
use wasmtime_environ::{
2929
BuiltinFunctionIndex, ComponentPC, DataIndex, DefinedFuncIndex, ElemIndex,
3030
EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey,
31-
GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, Module, ModuleInternedTypeIndex,
32-
ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex, TagIndex, Tunables,
33-
TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType,
34-
WasmHeapType, WasmRefType, WasmResult, WasmValType,
31+
GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module,
32+
ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex,
33+
TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,
34+
WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmValType,
3535
};
3636
use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};
3737

@@ -1566,6 +1566,7 @@ impl FuncEnvironment<'_> {
15661566
self.heaps.push(HeapData {
15671567
base,
15681568
bound,
1569+
kind: MemoryKind::LinearMemory,
15691570
memory,
15701571
})
15711572
}
@@ -1578,9 +1579,10 @@ impl FuncEnvironment<'_> {
15781579
offset: i32,
15791580
) -> ir::GlobalValue {
15801581
let pointer_type = self.pointer_type();
1582+
let memory_tunables = MemoryTunables::new(self.tunables, MemoryKind::LinearMemory);
15811583

15821584
let mut flags = ir::MemFlags::trusted().with_can_move();
1583-
if !memory.memory_may_move(self.tunables) {
1585+
if !memory.memory_may_move(&memory_tunables) {
15841586
flags.set_readonly();
15851587
}
15861588

crates/cranelift/src/func_environ/gc/enabled.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use super::{ArrayInit, GcCompiler};
22
use crate::bounds_checks::BoundsCheck;
33
use crate::func_environ::{Extension, FuncEnvironment};
4-
use crate::translate::{Heap, HeapData, StructFieldsVec, TargetEnvironment};
4+
use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment};
55
use crate::trap::TranslateTrap;
66
use crate::{Reachability, TRAP_INTERNAL_ASSERT};
77
use cranelift_codegen::ir::immediates::Offset32;
@@ -1449,10 +1449,12 @@ impl FuncEnvironment<'_> {
14491449
let offset = self.offsets.ptr.vmstore_context_gc_heap_base();
14501450

14511451
let mut flags = ir::MemFlags::trusted();
1452+
let memory_tunables =
1453+
wasmtime_environ::MemoryTunables::new(self.tunables, MemoryKind::GcHeap);
14521454
if !self
14531455
.tunables
14541456
.gc_heap_memory_type()
1455-
.memory_may_move(self.tunables)
1457+
.memory_may_move(&memory_tunables)
14561458
{
14571459
flags.set_readonly();
14581460
flags.set_can_move();
@@ -1512,6 +1514,7 @@ impl FuncEnvironment<'_> {
15121514
base,
15131515
bound,
15141516
memory,
1517+
kind: MemoryKind::GcHeap,
15151518
});
15161519
self.gc_heap = Some(heap);
15171520
heap

crates/cranelift/src/translate/heap.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ use cranelift_codegen::ir::{self, GlobalValue, Type};
44
use cranelift_entity::entity_impl;
55
use wasmtime_environ::{IndexType, Memory};
66

7+
pub use wasmtime_environ::MemoryKind;
8+
79
/// An opaque reference to a [`HeapData`][crate::HeapData].
810
///
911
/// While the order is stable, it is arbitrary.
@@ -45,6 +47,9 @@ pub struct HeapData {
4547

4648
/// The type of wasm memory that this heap is operating on.
4749
pub memory: Memory,
50+
51+
/// Whether this is a linear memory or a GC heap.
52+
pub kind: MemoryKind,
4853
}
4954

5055
impl HeapData {
@@ -54,4 +59,12 @@ impl HeapData {
5459
IndexType::I64 => ir::types::I64,
5560
}
5661
}
62+
63+
/// Get the [`MemoryTunables`] for this heap based on its [`MemoryKind`].
64+
pub fn memory_tunables<'a>(
65+
&self,
66+
tunables: &'a wasmtime_environ::Tunables,
67+
) -> wasmtime_environ::MemoryTunables<'a> {
68+
wasmtime_environ::MemoryTunables::new(tunables, self.kind)
69+
}
5770
}

crates/cranelift/src/translate/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ mod translation_utils;
2020

2121
pub use self::environ::{GlobalVariable, StructFieldsVec, TargetEnvironment};
2222
pub use self::func_translator::FuncTranslator;
23-
pub use self::heap::{Heap, HeapData};
23+
pub use self::heap::{Heap, HeapData, MemoryKind};
2424
pub use self::stack::FuncTranslationStacks;
2525
pub use self::table::{TableData, TableSize};
2626
pub use self::translation_utils::*;

crates/environ/src/tunables.rs

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,29 @@ define_tunables! {
160160
/// forced and the counter is reset. Only effective when
161161
/// `cfg(gc_zeal)` is enabled.
162162
pub gc_zeal_alloc_counter: Option<NonZeroU32>,
163+
164+
/// Initial size, in bytes, to be allocated for GC heaps.
165+
///
166+
/// This is the same as `memory_reservation` but for GC heaps.
167+
pub gc_heap_reservation: u64,
168+
169+
/// The size, in bytes, of the guard page region for GC heaps.
170+
///
171+
/// This is the same as `memory_guard_size` but for GC heaps.
172+
pub gc_heap_guard_size: u64,
173+
174+
/// The size, in bytes, to allocate at the end of a relocated GC heap
175+
/// for growth.
176+
///
177+
/// This is the same as `memory_reservation_for_growth` but for GC
178+
/// heaps.
179+
pub gc_heap_reservation_for_growth: u64,
180+
181+
/// Whether or not GC heaps are allowed to be reallocated after initial
182+
/// allocation at runtime.
183+
///
184+
/// This is the same as `memory_may_move` but for GC heaps.
185+
pub gc_heap_may_move: bool,
163186
}
164187

165188
pub struct ConfigTunables {
@@ -200,6 +223,7 @@ impl Tunables {
200223
if target.is_pulley() {
201224
ret.signals_based_traps = false;
202225
ret.memory_guard_size = 0;
226+
ret.gc_heap_guard_size = 0;
203227
}
204228
Ok(ret)
205229
}
@@ -239,6 +263,10 @@ impl Tunables {
239263
concurrency_support: true,
240264
recording: false,
241265
gc_zeal_alloc_counter: None,
266+
gc_heap_reservation: 0,
267+
gc_heap_guard_size: 0,
268+
gc_heap_reservation_for_growth: 0,
269+
gc_heap_may_move: true,
242270
}
243271
}
244272

@@ -253,6 +281,12 @@ impl Tunables {
253281
memory_reservation_for_growth: 1 << 20, // 1MB
254282
signals_based_traps: true,
255283

284+
// GC heaps on 32-bit: conservative defaults similar to linear
285+
// memories.
286+
gc_heap_reservation: 10 * (1 << 20),
287+
gc_heap_guard_size: 0x1_0000,
288+
gc_heap_reservation_for_growth: 1 << 20, // 1MB
289+
256290
..Tunables::default_miri()
257291
}
258292
}
@@ -278,6 +312,12 @@ impl Tunables {
278312
// to avoid memory movement.
279313
memory_reservation_for_growth: 2 << 30, // 2GB
280314

315+
// GC heaps on 64-bit: use 4GiB reservation and 32MiB guard pages
316+
// to enable bounds check elision, matching linear memory defaults.
317+
gc_heap_reservation: 1 << 32,
318+
gc_heap_guard_size: 32 << 20,
319+
gc_heap_reservation_for_growth: 2 << 30, // 2GB
320+
281321
signals_based_traps: true,
282322
..Tunables::default_miri()
283323
}
@@ -298,6 +338,73 @@ impl Tunables {
298338
}
299339
}
300340

341+
/// Whether a heap is backing a linear memory or a GC heap.
342+
///
343+
/// This is used by [`MemoryTunables`] to select between the memory tunables and
344+
/// the GC heap tunables.
345+
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
346+
pub enum MemoryKind {
347+
/// A WebAssembly linear memory.
348+
LinearMemory,
349+
/// A GC heap for garbage-collected objects.
350+
GcHeap,
351+
}
352+
353+
/// A view into a [`Tunables`] that selects the appropriate linear-memory or
354+
/// GC-heap flavor of each tunable based on a [`MemoryKind`].
355+
pub struct MemoryTunables<'a> {
356+
tunables: &'a Tunables,
357+
kind: MemoryKind,
358+
}
359+
360+
impl<'a> MemoryTunables<'a> {
361+
/// Create a new `MemoryTunables` view.
362+
pub fn new(tunables: &'a Tunables, kind: MemoryKind) -> Self {
363+
Self { tunables, kind }
364+
}
365+
366+
/// The virtual memory reservation for this kind of memory.
367+
pub fn reservation(&self) -> u64 {
368+
match self.kind {
369+
MemoryKind::LinearMemory => self.tunables.memory_reservation,
370+
MemoryKind::GcHeap => self.tunables.gc_heap_reservation,
371+
}
372+
}
373+
374+
/// The size of the guard page region for this kind of memory.
375+
pub fn guard_size(&self) -> u64 {
376+
match self.kind {
377+
MemoryKind::LinearMemory => self.tunables.memory_guard_size,
378+
MemoryKind::GcHeap => self.tunables.gc_heap_guard_size,
379+
}
380+
}
381+
382+
/// Extra virtual memory to reserve beyond the initially mapped pages for
383+
/// this kind of memory.
384+
pub fn reservation_for_growth(&self) -> u64 {
385+
match self.kind {
386+
MemoryKind::LinearMemory => self.tunables.memory_reservation_for_growth,
387+
MemoryKind::GcHeap => self.tunables.gc_heap_reservation_for_growth,
388+
}
389+
}
390+
391+
/// Whether this kind of memory's base pointer may be relocated at runtime.
392+
pub fn may_move(&self) -> bool {
393+
match self.kind {
394+
MemoryKind::LinearMemory => self.tunables.memory_may_move,
395+
MemoryKind::GcHeap => self.tunables.gc_heap_may_move,
396+
}
397+
}
398+
399+
/// Get the underlying tunables.
400+
///
401+
/// This is ONLY for accessing tunable fields that DO NOT come in a
402+
/// linear-memory flavor and a GC-heap flavor.
403+
pub fn tunables(&self) -> &'a Tunables {
404+
self.tunables
405+
}
406+
}
407+
301408
/// The garbage collector implementation to use.
302409
#[derive(Clone, Copy, Hash, Serialize, Deserialize, Debug, PartialEq, Eq)]
303410
pub enum Collector {

0 commit comments

Comments
 (0)