From 28ac9d09f01538845454def466c03d53fb6a791c Mon Sep 17 00:00:00 2001 From: Matt Paras Date: Mon, 26 May 2025 20:21:29 -0700 Subject: [PATCH 1/2] checkpoint --- crates/steel-core/src/compiler/compiler.rs | 2 +- crates/steel-core/src/compiler/constants.rs | 12 +- crates/steel-core/src/rvals.rs | 115 ++- crates/steel-core/src/steel_vm/builtin.rs | 15 +- crates/steel-core/src/steel_vm/cache.rs | 10 +- crates/steel-core/src/steel_vm/primitives.rs | 120 +++- crates/steel-core/src/steel_vm/vm.rs | 18 + crates/steel-core/src/steel_vm/vm/threads.rs | 700 ++++++++++--------- crates/steel-core/src/values/functions.rs | 8 +- crates/steel-core/src/values/lazy_stream.rs | 8 +- crates/steel-core/src/values/structs.rs | 9 +- 11 files changed, 612 insertions(+), 405 deletions(-) diff --git a/crates/steel-core/src/compiler/compiler.rs b/crates/steel-core/src/compiler/compiler.rs index 72559bc73..e77e262ab 100644 --- a/crates/steel-core/src/compiler/compiler.rs +++ b/crates/steel-core/src/compiler/compiler.rs @@ -336,7 +336,7 @@ pub struct Compiler { // is under the hood, shared references to the engine, since we // want to have the compiler share everything with the runtime. sources: Sources, - builtin_modules: ModuleContainer, + pub(crate) builtin_modules: ModuleContainer, } pub struct SerializableCompiler { diff --git a/crates/steel-core/src/compiler/constants.rs b/crates/steel-core/src/compiler/constants.rs index cc230ad47..f244666a7 100644 --- a/crates/steel-core/src/compiler/constants.rs +++ b/crates/steel-core/src/compiler/constants.rs @@ -1,7 +1,9 @@ use crate::gc::shared::{MutContainer, ShareableMut}; use crate::gc::{Shared, SharedMut}; use crate::parser::tryfrom_visitor::TryFromExprKindForSteelVal; -use crate::rvals::{into_serializable_value, Result, SerializableSteelVal, SteelVal}; +use crate::rvals::{ + into_serializable_value, Result, SerializableSteelVal, SerializationContext, SteelVal, +}; use crate::parser::{ ast::ExprKind, @@ -84,16 +86,12 @@ impl ConstantMap { SerializableConstantMap(self.to_bytes().unwrap()) } - pub fn to_serializable_vec( - &self, - serializer: &mut std::collections::HashMap, - visited: &mut std::collections::HashSet, - ) -> Vec { + pub fn to_serializable_vec(&self, ctx: &mut SerializationContext) -> Vec { self.values .read() .iter() .cloned() - .map(|x| into_serializable_value(x, serializer, visited)) + .map(|x| into_serializable_value(x, ctx)) .collect::>() .unwrap() } diff --git a/crates/steel-core/src/rvals.rs b/crates/steel-core/src/rvals.rs index df0d083ea..75e48714c 100644 --- a/crates/steel-core/src/rvals.rs +++ b/crates/steel-core/src/rvals.rs @@ -17,11 +17,14 @@ use crate::{ }, primitives::numbers::realp, rerrs::{ErrorKind, SteelErr}, - steel_vm::vm::{threads::closure_into_serializable, BuiltInSignature, Continuation}, + steel_vm::{ + engine::ModuleContainer, + vm::{threads::closure_into_serializable, BuiltInSignature, Continuation}, + }, values::{ closed::{Heap, HeapRef, MarkAndSweepContext}, functions::{BoxedDynFunction, ByteCodeLambda}, - lazy_stream::LazyStream, + lazy_stream::{LazyStream, SerializableStream}, port::{SendablePort, SteelPort}, structs::{SerializableUserDefinedStruct, UserDefinedStruct}, transducers::{Reducer, Transducer}, @@ -847,6 +850,7 @@ pub enum SerializableSteelVal { FuncV(FunctionSignature), MutFunc(MutFunctionSignature), HashMapV(Vec<(SerializableSteelVal, SerializableSteelVal)>), + HashSet(Vec), ListV(Vec), Pair(Box<(SerializableSteelVal, SerializableSteelVal)>), VectorV(Vec), @@ -860,6 +864,25 @@ pub enum SerializableSteelVal { HeapAllocated(usize), Port(SendablePort), Rational(Rational32), + Stream(Box), + NativeRef(NativeRefSpec), +} + +/* +Serialize via known modules. If the value is a native one, then it should get replaced with a +value that is something like SerializableSteelVal::NativeRef(NativeRefSpec) where NativeRefSpec is like: + +struct NativeRefSpec { + module: String, + key: String, +} + +And then deserializing is just grabbing that back from the root. +*/ + +pub struct NativeRefSpec { + pub module: String, + pub key: String, } pub enum SerializedHeapRef { @@ -921,6 +944,14 @@ pub fn from_serializable_value(ctx: &mut HeapSerializer, val: SerializableSteelV ) .into(), ), + SerializableSteelVal::HashSet(h) => SteelVal::HashSetV( + Gc::new( + h.into_iter() + .map(|k| from_serializable_value(ctx, k)) + .collect::>(), + ) + .into(), + ), SerializableSteelVal::ListV(v) => SteelVal::ListV( v.into_iter() .map(|x| from_serializable_value(ctx, x)) @@ -1013,38 +1044,63 @@ pub fn from_serializable_value(ctx: &mut HeapSerializer, val: SerializableSteelV SerializableSteelVal::ByteVectorV(bytes) => { SteelVal::ByteVector(SteelByteVector::new(bytes)) } + + SerializableSteelVal::Stream(value) => SteelVal::StreamV(Gc::new(LazyStream { + initial_value: from_serializable_value(ctx, value.initial_value), + stream_thunk: from_serializable_value(ctx, value.stream_thunk), + empty_stream: value.empty_stream, + })), + + SerializableSteelVal::NativeRef(_) => { + todo!("Implement native ref spec deserialization!") + } } } // The serializable value needs to refer to the original heap - // that way can reference the original stuff easily. +pub struct SerializationContext<'a> { + pub builtin_modules: &'a ModuleContainer, + pub serialized_heap: &'a mut std::collections::HashMap, + pub visited: &'a mut std::collections::HashSet, +} + // TODO: Use the cycle detector instead pub fn into_serializable_value( val: SteelVal, - serialized_heap: &mut std::collections::HashMap, - visited: &mut std::collections::HashSet, + ctx: &mut SerializationContext, + // serialized_heap: &mut std::collections::HashMap, + // visited: &mut std::collections::HashSet, ) -> Result { // dbg!(&serialized_heap); match val { - SteelVal::Closure(c) => closure_into_serializable(&c, serialized_heap, visited) - .map(SerializableSteelVal::Closure), + SteelVal::Closure(c) => { + closure_into_serializable(&c, ctx).map(SerializableSteelVal::Closure) + } SteelVal::BoolV(b) => Ok(SerializableSteelVal::BoolV(b)), SteelVal::NumV(n) => Ok(SerializableSteelVal::NumV(n)), SteelVal::IntV(n) => Ok(SerializableSteelVal::IntV(n)), SteelVal::CharV(c) => Ok(SerializableSteelVal::CharV(c)), SteelVal::Void => Ok(SerializableSteelVal::Void), SteelVal::StringV(s) => Ok(SerializableSteelVal::StringV(s.to_string())), - SteelVal::FuncV(f) => Ok(SerializableSteelVal::FuncV(f)), + // SteelVal::FuncV(f) => Ok(SerializableSteelVal::FuncV(f)), + SteelVal::FuncV(f) => Ok(SerializableSteelVal::NativeRef( + // TODO: Native ref spec for anything that is native + // and truly can't be serialized between runtimes, such as native + // functions. + crate::steel_vm::vm::threads::create_native_ref(&ctx.builtin_modules, val.clone()) + .expect(&format!("Unable to find: {}", val)), + )), SteelVal::ListV(l) => Ok(SerializableSteelVal::ListV( l.into_iter() - .map(|x| into_serializable_value(x, serialized_heap, visited)) + .map(|x| into_serializable_value(x, ctx)) .collect::>()?, )), SteelVal::Pair(pair) => Ok(SerializableSteelVal::Pair(Box::new(( - into_serializable_value(pair.car.clone(), serialized_heap, visited)?, - into_serializable_value(pair.cdr.clone(), serialized_heap, visited)?, + into_serializable_value(pair.car.clone(), ctx)?, + into_serializable_value(pair.cdr.clone(), ctx)?, )))), SteelVal::BoxedFunction(f) => Ok(SerializableSteelVal::BoxedDynFunction((*f).clone())), SteelVal::BuiltIn(f) => Ok(SerializableSteelVal::BuiltIn(f)), @@ -1054,8 +1110,8 @@ pub fn into_serializable_value( v.0.unwrap() .into_iter() .map(|(k, v)| { - let kprime = into_serializable_value(k, serialized_heap, visited)?; - let vprime = into_serializable_value(v, serialized_heap, visited)?; + let kprime = into_serializable_value(k, ctx)?; + let vprime = into_serializable_value(v, ctx)?; Ok((kprime, vprime)) }) @@ -1063,10 +1119,12 @@ pub fn into_serializable_value( )), SteelVal::Custom(c) => { - if let Some(output) = c.write().as_serializable_steelval() { + let mut guard = c.write(); + + if let Some(output) = guard.as_serializable_steelval() { Ok(output) } else { - stop!(Generic => "Custom type not allowed to be moved across threads!") + stop!(Generic => "Custom type not allowed to be moved across threads!: {}", guard.name()) } } @@ -1076,7 +1134,7 @@ pub fn into_serializable_value( .fields .iter() .cloned() - .map(|x| into_serializable_value(x, serialized_heap, visited)) + .map(|x| into_serializable_value(x, ctx)) .collect::>>()?, type_descriptor: s.type_descriptor, }, @@ -1087,23 +1145,23 @@ pub fn into_serializable_value( // If there is a cycle, this could cause problems? SteelVal::HeapAllocated(h) => { // We should pick it up on the way back the recursion - if visited.contains(&h.as_ptr_usize()) - && !serialized_heap.contains_key(&h.as_ptr_usize()) + if ctx.visited.contains(&h.as_ptr_usize()) + && !ctx.serialized_heap.contains_key(&h.as_ptr_usize()) { // println!("Already visited: {}", h.as_ptr_usize()); Ok(SerializableSteelVal::HeapAllocated(h.as_ptr_usize())) } else { - visited.insert(h.as_ptr_usize()); + ctx.visited.insert(h.as_ptr_usize()); - if serialized_heap.contains_key(&h.as_ptr_usize()) { + if ctx.serialized_heap.contains_key(&h.as_ptr_usize()) { // println!("Already exists in map: {}", h.as_ptr_usize()); Ok(SerializableSteelVal::HeapAllocated(h.as_ptr_usize())) } else { // println!("Trying to insert: {} @ {}", h.get(), h.as_ptr_usize()); - let value = into_serializable_value(h.get(), serialized_heap, visited); + let value = into_serializable_value(h.get(), ctx); let value = match value { Ok(v) => v, @@ -1113,7 +1171,7 @@ pub fn into_serializable_value( } }; - serialized_heap.insert(h.as_ptr_usize(), value); + ctx.serialized_heap.insert(h.as_ptr_usize(), value); // println!("Inserting: {}", h.as_ptr_usize()); @@ -1126,7 +1184,7 @@ pub fn into_serializable_value( vector .iter() .cloned() - .map(|val| into_serializable_value(val, serialized_heap, visited)) + .map(|val| into_serializable_value(val, ctx)) .collect::>()?, )), @@ -1136,6 +1194,19 @@ pub fn into_serializable_value( SteelVal::Rational(r) => Ok(SerializableSteelVal::Rational(r)), + SteelVal::StreamV(s) => Ok(SerializableSteelVal::Stream(Box::new(SerializableStream { + initial_value: into_serializable_value(s.initial_value.clone(), ctx)?, + stream_thunk: into_serializable_value(s.stream_thunk.clone(), ctx)?, + empty_stream: s.empty_stream, + }))), + + SteelVal::HashSetV(s) => Ok(SerializableSteelVal::HashSet( + s.iter() + .cloned() + .map(|val| into_serializable_value(val, ctx)) + .collect::>()?, + )), + illegal => stop!(Generic => "Type not allowed to be moved across threads!: {}", illegal), } } diff --git a/crates/steel-core/src/steel_vm/builtin.rs b/crates/steel-core/src/steel_vm/builtin.rs index e21213342..c3e3f72b0 100644 --- a/crates/steel-core/src/steel_vm/builtin.rs +++ b/crates/steel-core/src/steel_vm/builtin.rs @@ -115,7 +115,13 @@ impl Custom for FunctionSignatureMetadata { } } -impl Custom for BuiltInModule {} +impl Custom for BuiltInModule { + fn into_serializable_steelval(&mut self) -> Option { + Some(crate::rvals::SerializableSteelVal::Custom(Box::new( + self.clone(), + ))) + } +} impl RegisterValue for BuiltInModule { fn register_value_inner(&mut self, name: &str, value: SteelVal) -> &mut Self { @@ -540,6 +546,13 @@ impl BuiltInModule { self.module.read().names() } + pub(crate) fn inner_map( + &self, + ) -> MappedScopedReadContainer<'_, std::collections::HashMap, SteelVal, FxBuildHasher>> + { + ScopedReadContainer::map(self.module.read(), |x| &x.values) + } + pub fn name(&self) -> Shared { Shared::clone(&self.module.read().name) } diff --git a/crates/steel-core/src/steel_vm/cache.rs b/crates/steel-core/src/steel_vm/cache.rs index cc9a0403b..398b802ba 100644 --- a/crates/steel-core/src/steel_vm/cache.rs +++ b/crates/steel-core/src/steel_vm/cache.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use crate::values::lists::List; +use crate::{rvals::SerializableSteelVal, values::lists::List}; use weak_table::WeakKeyHashMap; use crate::{rvals::Custom, values::functions::ByteCodeLambda, SteelVal}; @@ -44,6 +44,7 @@ impl MemoizationTable { } } +#[derive(Clone)] pub struct WeakMemoizationTable { #[cfg(not(feature = "sync"))] table: WeakKeyHashMap, HashMap, SteelVal>>, @@ -96,4 +97,9 @@ impl WeakMemoizationTable { } } -impl Custom for WeakMemoizationTable {} +#[cfg(feature = "sync")] +impl Custom for WeakMemoizationTable { + fn into_serializable_steelval(&mut self) -> Option { + Some(SerializableSteelVal::Custom(Box::new(self.clone()))) + } +} diff --git a/crates/steel-core/src/steel_vm/primitives.rs b/crates/steel-core/src/steel_vm/primitives.rs index fb10ffec8..765586eed 100644 --- a/crates/steel-core/src/steel_vm/primitives.rs +++ b/crates/steel-core/src/steel_vm/primitives.rs @@ -4,10 +4,11 @@ use super::{ engine::Engine, register_fn::RegisterFn, vm::{ - get_test_mode, list_modules, set_test_mode, VmCore, CALL_CC_DEFINITION, - CALL_WITH_EXCEPTION_HANDLER_DEFINITION, EVAL_DEFINITION, EVAL_FILE_DEFINITION, - EVAL_STRING_DEFINITION, EXPAND_SYNTAX_CASE_DEFINITION, EXPAND_SYNTAX_OBJECTS_DEFINITION, - INSPECT_DEFINITION, MACRO_CASE_BINDINGS_DEFINITION, MATCH_SYNTAX_CASE_DEFINITION, + get_test_mode, list_modules, set_test_mode, threads::SERIALIZE_THREAD_DEFINITION, VmCore, + CALL_CC_DEFINITION, CALL_WITH_EXCEPTION_HANDLER_DEFINITION, DEBUG_GLOBALS_DEFINITION, + EVAL_DEFINITION, EVAL_FILE_DEFINITION, EVAL_STRING_DEFINITION, + EXPAND_SYNTAX_CASE_DEFINITION, EXPAND_SYNTAX_OBJECTS_DEFINITION, INSPECT_DEFINITION, + MACRO_CASE_BINDINGS_DEFINITION, MATCH_SYNTAX_CASE_DEFINITION, }, }; use crate::{ @@ -488,47 +489,97 @@ fn render_as_md(text: String) { println!("{}", text); } -pub fn register_builtin_modules(engine: &mut Engine, sandbox: bool) { - engine.register_value("std::env::args", SteelVal::ListV(List::new())); +pub fn private_prim_module() -> BuiltInModule { + let mut module = BuiltInModule::new("#%private/steel/primitives"); + + module.register_value("std::env::args", SteelVal::ListV(List::new())); - engine.register_fn("##__module-get", BuiltInModule::get); - engine.register_fn("%module-get%", BuiltInModule::get); - engine.register_fn("%#maybe-module-get", BuiltInModule::try_get); + module.register_fn("##__module-get", BuiltInModule::get); + module.register_fn("%module-get%", BuiltInModule::get); + module.register_fn("%#maybe-module-get", BuiltInModule::try_get); - engine.register_fn("load-from-module!", BuiltInModule::get); + module.register_fn("load-from-module!", BuiltInModule::get); // Registering values in modules - engine.register_fn("#%module", BuiltInModule::new::); - engine.register_fn( + module.register_fn("#%module", BuiltInModule::new::); + module.register_fn( "#%module-add", |module: &mut BuiltInModule, name: SteelString, value: SteelVal| { module.register_value(&name, value); }, ); - engine.register_fn("%doc?", BuiltInModule::get_doc); - engine.register_value("%list-modules!", SteelVal::BuiltIn(list_modules)); - engine.register_fn("%module/lookup-function", BuiltInModule::search); - engine.register_fn("%string->render-markdown", render_as_md); - engine.register_fn( + module.register_fn("%doc?", BuiltInModule::get_doc); + module.register_value("%list-modules!", SteelVal::BuiltIn(list_modules)); + module.register_fn("%module/lookup-function", BuiltInModule::search); + module.register_fn("%string->render-markdown", render_as_md); + module.register_fn( "%module-bound-identifiers->list", BuiltInModule::bound_identifiers, ); - engine.register_value("%proto-hash%", HM_CONSTRUCT); - engine.register_value("%proto-hash-insert%", HM_INSERT); - engine.register_value("%proto-hash-get%", HM_GET); - engine.register_value("error!", ControlOperations::error()); - - engine.register_value("error", ControlOperations::error()); - - engine.register_value("#%error", ControlOperations::error()); - - engine.register_value( + module.register_value("%proto-hash%", HM_CONSTRUCT); + module.register_value("%proto-hash-insert%", HM_INSERT); + module.register_value("%proto-hash-get%", HM_GET); + module.register_value("error!", ControlOperations::error()); + module.register_value("error", ControlOperations::error()); + module.register_value("#%error", ControlOperations::error()); + + module.register_value( "%memo-table", WeakMemoizationTable::new().into_steelval().unwrap(), ); - engine.register_fn("%memo-table-ref", WeakMemoizationTable::get); - engine.register_fn("%memo-table-set!", WeakMemoizationTable::insert); + module.register_fn("%memo-table-ref", WeakMemoizationTable::get); + module.register_fn("%memo-table-set!", WeakMemoizationTable::insert); + + module +} + +pub fn register_builtin_modules(engine: &mut Engine, sandbox: bool) { + let prims = private_prim_module(); + engine.register_module(prims.clone()); + + for (key, value) in prims.inner_map().iter() { + engine.register_value(key, value.clone()); + } + + // engine.register_value("std::env::args", SteelVal::ListV(List::new())); + + // engine.register_fn("##__module-get", BuiltInModule::get); + // engine.register_fn("%module-get%", BuiltInModule::get); + // engine.register_fn("%#maybe-module-get", BuiltInModule::try_get); + + // engine.register_fn("load-from-module!", BuiltInModule::get); + + // // Registering values in modules + // engine.register_fn("#%module", BuiltInModule::new::); + // engine.register_fn( + // "#%module-add", + // |module: &mut BuiltInModule, name: SteelString, value: SteelVal| { + // module.register_value(&name, value); + // }, + // ); + + // engine.register_fn("%doc?", BuiltInModule::get_doc); + // engine.register_value("%list-modules!", SteelVal::BuiltIn(list_modules)); + // engine.register_fn("%module/lookup-function", BuiltInModule::search); + // engine.register_fn("%string->render-markdown", render_as_md); + // engine.register_fn( + // "%module-bound-identifiers->list", + // BuiltInModule::bound_identifiers, + // ); + // engine.register_value("%proto-hash%", HM_CONSTRUCT); + // engine.register_value("%proto-hash-insert%", HM_INSERT); + // engine.register_value("%proto-hash-get%", HM_GET); + // engine.register_value("error!", ControlOperations::error()); + // engine.register_value("error", ControlOperations::error()); + // engine.register_value("#%error", ControlOperations::error()); + + // engine.register_value( + // "%memo-table", + // WeakMemoizationTable::new().into_steelval().unwrap(), + // ); + // engine.register_fn("%memo-table-ref", WeakMemoizationTable::get); + // engine.register_fn("%memo-table-set!", WeakMemoizationTable::insert); #[cfg(feature = "sync")] { @@ -1551,12 +1602,19 @@ impl crate::rvals::Custom for MutableVector { } } +#[derive(Clone)] struct Reader { buffer: String, offset: usize, } -impl crate::rvals::Custom for Reader {} +impl crate::rvals::Custom for Reader { + fn into_serializable_steelval(&mut self) -> Option { + Some(crate::rvals::SerializableSteelVal::Custom(Box::new( + self.clone(), + ))) + } +} impl Reader { fn create_reader() -> Reader { @@ -1773,6 +1831,8 @@ fn meta_module() -> BuiltInModule { .register_value("raise-error", raise_error_from_error()) .register_native_fn_definition(CALL_CC_DEFINITION) .register_native_fn_definition(EVAL_DEFINITION) + .register_native_fn_definition(DEBUG_GLOBALS_DEFINITION) + .register_native_fn_definition(SERIALIZE_THREAD_DEFINITION) .register_native_fn_definition(EVAL_FILE_DEFINITION) .register_native_fn_definition(EXPAND_SYNTAX_OBJECTS_DEFINITION) .register_native_fn_definition(MATCH_SYNTAX_CASE_DEFINITION) diff --git a/crates/steel-core/src/steel_vm/vm.rs b/crates/steel-core/src/steel_vm/vm.rs index eb0b2da5a..72b3ccb18 100644 --- a/crates/steel-core/src/steel_vm/vm.rs +++ b/crates/steel-core/src/steel_vm/vm.rs @@ -4625,6 +4625,24 @@ pub fn call_cc(ctx: &mut VmCore, args: &[SteelVal]) -> Option> Some(Ok(SteelVal::ContinuationFunction(continuation))) } +#[steel_derive::context(name = "debug-globals", arity = "Exact(0)")] +fn debug_globals(ctx: &mut VmCore, args: &[SteelVal]) -> Option> { + Some(debug_global_env(ctx, args)) +} + +fn debug_global_env(ctx: &mut VmCore, args: &[SteelVal]) -> Result { + let compiler = ctx.thread.compiler.read(); + + let symbols = compiler.symbol_map.values(); + let globals = &ctx.thread.global_env.thread_local_bindings; + + for (key, value) in symbols.iter().zip(globals.iter()) { + println!("{} = {}", key, value); + } + + Ok(SteelVal::Void) +} + fn eval_impl(ctx: &mut crate::steel_vm::vm::VmCore, args: &[SteelVal]) -> Result { let expr = crate::parser::ast::TryFromSteelValVisitorForExprKind::root(&args[0])?; // TODO: Looks like this isn't correctly parsing / pushing down macros! diff --git a/crates/steel-core/src/steel_vm/vm/threads.rs b/crates/steel-core/src/steel_vm/vm/threads.rs index c00e3370d..a0f08845b 100644 --- a/crates/steel-core/src/steel_vm/vm/threads.rs +++ b/crates/steel-core/src/steel_vm/vm/threads.rs @@ -4,9 +4,12 @@ use fxhash::FxHashMap; use steel_derive::function; use crate::{ - rvals::{AsRefMutSteelVal, AsRefSteelVal as _, Custom, HeapSerializer, SerializableSteelVal}, - steel_vm::{builtin::BuiltInModule, register_fn::RegisterFn}, - values::functions::SerializedLambdaPrototype, + rvals::{ + from_serializable_value, AsRefMutSteelVal, AsRefSteelVal as _, Custom, HeapSerializer, + NativeRefSpec, SerializableSteelVal, SerializationContext, SerializedHeapRef, + }, + steel_vm::{builtin::BuiltInModule, engine::ModuleContainer, register_fn::RegisterFn}, + values::{functions::SerializedLambdaPrototype, structs::VTable}, }; use super::*; @@ -159,8 +162,7 @@ thread_local! { pub fn closure_into_serializable( c: &ByteCodeLambda, - serializer: &mut std::collections::HashMap, - visited: &mut std::collections::HashSet, + ctx: &mut SerializationContext, ) -> Result { if let Some(prototype) = CACHED_CLOSURES.with(|x| x.borrow().get(&c.id).cloned()) { let mut prototype = SerializedLambda { @@ -175,7 +177,7 @@ pub fn closure_into_serializable( .captures .iter() .cloned() - .map(|x| into_serializable_value(x, serializer, visited)) + .map(|x| into_serializable_value(x, ctx)) .collect::>()?; Ok(prototype) @@ -207,338 +209,358 @@ pub fn closure_into_serializable( .captures .iter() .cloned() - .map(|x| into_serializable_value(x, serializer, visited)) + .map(|x| into_serializable_value(x, ctx)) .collect::>()?; Ok(prototype) } } -// struct MovableThread { -// constants: Vec, -// global_env: Vec, -// function_interner: MovableFunctionInterner, -// runtime_options: RunTimeOptions, -// } +struct MovableThread { + constants: Vec, + global_env: Vec, + function_interner: MovableFunctionInterner, + runtime_options: RunTimeOptions, +} -// struct MovableFunctionInterner { -// closure_interner: fxhash::FxHashMap, -// pure_function_interner: fxhash::FxHashMap, -// spans: fxhash::FxHashMap>, -// } +struct MovableFunctionInterner { + closure_interner: fxhash::FxHashMap, + pure_function_interner: fxhash::FxHashMap, + spans: fxhash::FxHashMap>, +} + +#[steel_derive::context(name = "serialize-thread", arity = "Exact(0)")] +fn serialize_thread(ctx: &mut VmCore, args: &[SteelVal]) -> Option> { + Some(serialize_thread_impl(ctx, args)) +} + +// Create a native ref spec from the builtin modules. +// This should in theory be all we need in order to then reconstruct this +// native value reference on the other side. +// +// When deconstructing the value, we'll attempt to find the key for it +// and provide the builtin modules. Assuming it exists, we can locate +// the value and move on with our lives. +pub(crate) fn create_native_ref(ctx: &ModuleContainer, v: SteelVal) -> Option { + // Not good, but lets just see if it even works: + let module_map = ctx.inner(); + + for (mkey, module) in module_map.iter() { + let map = module.inner_map(); + + for (key, value) in map.iter() { + // Just check each item, and drain from our list if its it? + + if value == &v { + return Some(NativeRefSpec { + module: mkey.to_string(), + key: key.to_string(), + }); + } + } + } + + None +} // #[allow(unused)] // This will naively deep clone the environment, by attempting to translate every value into a `SerializableSteelVal` // While this does work, it does result in a fairly hefty deep clone of the environment. It does _not_ smartly attempt // to keep track of what values this function could touch - rather it assumes every value is possible to be touched // by the child thread. -// fn spawn_thread_result(ctx: &mut VmCore, args: &[SteelVal]) -> Result { -// use crate::rvals::SerializableSteelVal; - -// #[cfg(feature = "profiling")] -// let now = std::time::Instant::now(); - -// // Need a new: -// // Stack -// // Heap -// // global env - This we can do (hopefully) lazily. Only clone the values that actually -// // get referenced. We can also just straight up reject any closures that cannot be moved -// // across threads - -// if args.len() != 1 { -// stop!(ArityMismatch => "spawn-thread! accepts one argument, found: {}", args.len()) -// } - -// let mut initial_map = HashMap::new(); -// let mut visited = HashSet::new(); - -// // If it is a native function, theres no reason we can't just call it on a new thread, most likely. -// // There might be some funny business with thread local values, but for now we'll just accept it. -// let function: SerializedLambda = match &args[0] { -// SteelVal::FuncV(f) => { -// let func = *f; - -// let handle = std::thread::spawn(move || func(&[]).map_err(|e| e.to_string())); - -// return ThreadHandle { -// handle: Some(handle), -// thread_state_manager: ThreadStateController::default(), -// } -// .into_steelval(); - -// // todo!() -// } -// SteelVal::MutFunc(f) => { -// let func = *f; - -// let handle = std::thread::spawn(move || func(&mut []).map_err(|e| e.to_string())); - -// return ThreadHandle { -// handle: Some(handle), -// thread_state_manager: ThreadStateController::default(), -// } -// .into_steelval(); -// } - -// // Probably rename unwrap to something else -// SteelVal::Closure(f) => closure_into_serializable(&f, &mut initial_map, &mut visited)?, -// illegal => { -// stop!(TypeMismatch => "Cannot spawn value on another thread: {}", illegal); -// } -// }; - -// let constants = time!("Constant map serialization", { -// let constants = ctx -// .thread -// .constant_map -// .to_serializable_vec(&mut initial_map, &mut visited); - -// constants -// }); - -// let sources = ctx.thread.sources.clone(); - -// let thread = MovableThread { -// constants, - -// // Void in this case, is a poisoned value. We need to trace the closure -// // (and all of its references) - to find any / all globals that _could_ be -// // referenced. -// #[cfg(feature = "sync")] -// global_env: time!( -// "Global env serialization", -// ctx.thread -// .global_env -// .bindings_vec -// .read() -// .unwrap() -// .iter() -// .cloned() -// .map(|x| into_serializable_value(x, &mut initial_map, &mut visited)) -// .map(|x| x.unwrap_or(SerializableSteelVal::Void)) -// .collect() -// ), - -// #[cfg(not(feature = "sync"))] -// global_env: time!( -// "Global env serialization", -// ctx.thread -// .global_env -// .bindings_vec -// .iter() -// .cloned() -// .map(|x| into_serializable_value(x, &mut initial_map, &mut visited)) -// .map(|x| x.unwrap_or(SerializableSteelVal::Void)) -// .collect() -// ), - -// // Populate with the values after moving into the thread, spawn accordingly -// // TODO: Move this out of here -// function_interner: time!( -// "Function interner serialization", -// MovableFunctionInterner { -// closure_interner: ctx -// .thread -// .function_interner -// .closure_interner -// .iter() -// .map(|(k, v)| { -// let v_prime: SerializedLambda = -// closure_into_serializable(v, &mut initial_map, &mut visited) -// .expect("This shouldn't fail!"); -// (*k, v_prime) -// }) -// .collect(), -// pure_function_interner: ctx -// .thread -// .function_interner -// .pure_function_interner -// .iter() -// .map(|(k, v)| { -// let v_prime: SerializedLambda = -// closure_into_serializable(v, &mut initial_map, &mut visited) -// .expect("This shouldn't fail!"); -// (*k, v_prime) -// }) -// .collect(), -// spans: ctx -// .thread -// .function_interner -// .spans -// .iter() -// .map(|(k, v)| (*k, v.iter().copied().collect())) -// .collect(), -// } -// ), - -// runtime_options: ctx.thread.runtime_options.clone(), -// }; - -// let sendable_vtable_entries = VTable::sendable_entries(&mut initial_map, &mut visited)?; - -// // TODO: Spawn a bunch of threads at the start to handle requests. That way we don't need to do this -// // the whole time they're in there. -// let handle = std::thread::spawn(move || { -// let heap = time!("Heap Creation", Arc::new(Mutex::new(Heap::new()))); - -// // Move across threads? -// let mut mapping = initial_map -// .into_iter() -// .map(|(key, value)| (key, SerializedHeapRef::Serialized(Some(value)))) -// .collect(); - -// let mut patcher = HashMap::new(); -// let mut built_functions = HashMap::new(); - -// let mut heap_guard = heap.lock().unwrap(); - -// let mut serializer = HeapSerializer { -// heap: &mut heap_guard, -// fake_heap: &mut mapping, -// values_to_fill_in: &mut patcher, -// built_functions: &mut built_functions, -// }; - -// // Moved over the thread. We now have -// let closure: ByteCodeLambda = ByteCodeLambda::from_serialized(&mut serializer, function); - -// VTable::initialize_new_thread(sendable_vtable_entries, &mut serializer); - -// let constant_map = time!( -// "Constant map deserialization", -// ConstantMap::from_vec( -// thread -// .constants -// .into_iter() -// .map(|x| from_serializable_value(&mut serializer, x)) -// .collect(), -// ) -// ); - -// #[cfg(feature = "sync")] -// let global_env = time!( -// "Global env creation", -// Env { -// bindings_vec: Arc::new(std::sync::RwLock::new( -// thread -// .global_env -// .into_iter() -// .map(|x| from_serializable_value(&mut serializer, x)) -// .collect() -// )), -// // TODO: -// thread_local_bindings: Vec::new(), -// } -// ); - -// #[cfg(not(feature = "sync"))] -// let global_env = time!( -// "Global env creation", -// Env { -// bindings_vec: thread -// .global_env -// .into_iter() -// .map(|x| from_serializable_value(&mut serializer, x)) -// .collect(), -// } -// ); - -// let function_interner = time!( -// "Function interner time", -// FunctionInterner { -// closure_interner: thread -// .function_interner -// .closure_interner -// .into_iter() -// .map(|(k, v)| (k, ByteCodeLambda::from_serialized(&mut serializer, v))) -// .collect(), -// pure_function_interner: thread -// .function_interner -// .pure_function_interner -// .into_iter() -// .map(|(k, v)| ( -// k, -// if let Some(exists) = serializer.built_functions.get(&v.id) { -// exists.clone() -// } else { -// Gc::new(ByteCodeLambda::from_serialized(&mut serializer, v)) -// } -// )) -// .collect(), -// spans: thread -// .function_interner -// .spans -// .into_iter() -// .map(|(k, v)| (k, v.into())) -// .collect(), -// } -// ); - -// // Patch over the values in the final heap! - -// time!("Patching over heap values", { -// for (key, value) in serializer.values_to_fill_in { -// if let Some(cycled) = serializer.fake_heap.get(key) { -// match cycled { -// SerializedHeapRef::Serialized(_) => todo!(), -// // Patch over the cycle -// SerializedHeapRef::Closed(c) => { -// value.set(c.get()); -// } -// } -// } else { -// todo!() -// } -// } -// }); - -// drop(heap_guard); - -// // New thread! It will result in a run time error if the function references globals that cannot be shared -// // between threads. This is a bit of an unfortunate occurrence - we probably _should_ just have the engine share -// // as much as possible between threads. -// let mut thread = SteelThread { -// global_env, -// sources, -// stack: Vec::with_capacity(64), - -// #[cfg(feature = "dynamic")] -// profiler: OpCodeOccurenceProfiler::new(), - -// function_interner, -// heap, -// runtime_options: thread.runtime_options, -// current_frame: StackFrame::main(), -// stack_frames: Vec::with_capacity(32), -// constant_map, -// interrupted: Default::default(), -// synchronizer: Synchronizer::new(), -// thread_local_storage: Vec::new(), -// // TODO: Fix this -// compiler: todo!(), -// id: EngineId::new(), -// safepoints_enabled: false, -// }; +fn serialize_thread_impl(ctx: &mut VmCore, args: &[SteelVal]) -> Result { + // use crate::rvals::SerializableSteelVal; + + #[cfg(feature = "profiling")] + let now = std::time::Instant::now(); + + // Need a new: + // Stack + // Heap + // global env - This we can do (hopefully) lazily. Only clone the values that actually + // get referenced. We can also just straight up reject any closures that cannot be moved + // across threads + + let mut initial_map = HashMap::new(); + let mut visited = HashSet::new(); + + // If it is a native function, theres no reason we can't just call it on a new thread, most likely. + // There might be some funny business with thread local values, but for now we'll just accept it. + // let function: SerializedLambda = match &args[0] { + // SteelVal::FuncV(f) => { + // let func = *f; + + // let handle = std::thread::spawn(move || func(&[]).map_err(|e| e.to_string())); + + // return ThreadHandle { + // handle: Some(handle), + // thread_state_manager: ThreadStateController::default(), + // } + // .into_steelval(); + + // // todo!() + // } + // SteelVal::MutFunc(f) => { + // let func = *f; + + // let handle = std::thread::spawn(move || func(&mut []).map_err(|e| e.to_string())); + + // return ThreadHandle { + // handle: Some(handle), + // thread_state_manager: ThreadStateController::default(), + // } + // .into_steelval(); + // } + + // // Probably rename unwrap to something else + // SteelVal::Closure(f) => closure_into_serializable(&f, &mut initial_map, &mut visited)?, + // illegal => { + // stop!(TypeMismatch => "Cannot spawn value on another thread: {}", illegal); + // } + // }; + + let sources = ctx.thread.sources.clone(); + + let builtin_modules = ctx.thread.compiler.read().builtin_modules.clone(); + + let mut sctx = SerializationContext { + builtin_modules: &builtin_modules, + serialized_heap: &mut initial_map, + visited: &mut visited, + }; -// #[cfg(feature = "profiling")] -// log::info!(target: "threads", "Time taken to spawn thread: {:?}", now.elapsed()); - -// // Call the function! -// thread -// .call_function( -// thread.constant_map.clone(), -// SteelVal::Closure(Gc::new(closure)), -// Vec::new(), -// ) -// .map_err(|e| e.to_string()) -// }); - -// return ThreadHandle { -// handle: Some(handle), -// thread_state_manager: ThreadStateController::default(), -// } -// .into_steelval(); -// } + let constants = ctx.thread.constant_map.to_serializable_vec(&mut sctx); + + let thread = MovableThread { + constants, + + // Void in this case, is a poisoned value. We need to trace the closure + // (and all of its references) - to find any / all globals that _could_ be + // referenced. + #[cfg(feature = "sync")] + global_env: ctx + .thread + .global_env + .bindings_vec + .read() + .unwrap() + .iter() + .cloned() + .map(|x| into_serializable_value(x, &mut sctx).unwrap()) + .collect(), + + #[cfg(not(feature = "sync"))] + global_env: ctx + .thread + .global_env + .bindings_vec + .iter() + .cloned() + .map(|x| into_serializable_value(x, &mut sctx)) + .collect(), + + // Populate with the values after moving into the thread, spawn accordingly + // TODO: Move this out of here + function_interner: MovableFunctionInterner { + closure_interner: ctx + .thread + .function_interner + .closure_interner + .iter() + .map(|(k, v)| { + let v_prime: SerializedLambda = + closure_into_serializable(v, &mut sctx).expect("This shouldn't fail!"); + (*k, v_prime) + }) + .collect(), + pure_function_interner: ctx + .thread + .function_interner + .pure_function_interner + .iter() + .map(|(k, v)| { + let v_prime: SerializedLambda = + closure_into_serializable(v, &mut sctx).expect("This shouldn't fail!"); + (*k, v_prime) + }) + .collect(), + spans: ctx + .thread + .function_interner + .spans + .iter() + .map(|(k, v)| (*k, v.iter().copied().collect())) + .collect(), + }, + + runtime_options: ctx.thread.runtime_options.clone(), + }; + + let sendable_vtable_entries = VTable::sendable_entries(&mut sctx)?; + + // TODO: Spawn a bunch of threads at the start to handle requests. That way we don't need to do this + // the whole time they're in there. + let heap = Arc::new(Mutex::new(Heap::new())); + + // Move across threads? + let mut mapping = initial_map + .into_iter() + .map(|(key, value)| (key, SerializedHeapRef::Serialized(Some(value)))) + .collect(); + + let mut patcher = HashMap::new(); + let mut built_functions = HashMap::new(); + + let mut heap_guard = heap.lock().unwrap(); + + let mut serializer = HeapSerializer { + heap: &mut heap_guard, + fake_heap: &mut mapping, + values_to_fill_in: &mut patcher, + built_functions: &mut built_functions, + }; + + // Moved over the thread. We now have + // let closure: ByteCodeLambda = ByteCodeLambda::from_serialized(&mut serializer, function); + + VTable::initialize_new_thread(sendable_vtable_entries, &mut serializer); + + println!("Initialized vtable."); + + let constant_map = ConstantMap::from_vec( + thread + .constants + .into_iter() + .map(|x| from_serializable_value(&mut serializer, x)) + .collect(), + ); + + println!("Initialized constant map"); + + #[cfg(feature = "sync")] + let global_env = Env { + bindings_vec: Arc::new(std::sync::RwLock::new( + thread + .global_env + .into_iter() + .map(|x| from_serializable_value(&mut serializer, x)) + .collect(), + )), + // TODO: + thread_local_bindings: Vec::new(), + }; + + #[cfg(not(feature = "sync"))] + let global_env = Env { + bindings_vec: thread + .global_env + .into_iter() + .map(|x| from_serializable_value(&mut serializer, x)) + .collect(), + }; + + println!("Initialized global env"); + + let function_interner = FunctionInterner { + closure_interner: thread + .function_interner + .closure_interner + .into_iter() + .map(|(k, v)| (k, ByteCodeLambda::from_serialized(&mut serializer, v))) + .collect(), + pure_function_interner: thread + .function_interner + .pure_function_interner + .into_iter() + .map(|(k, v)| { + ( + k, + if let Some(exists) = serializer.built_functions.get(&v.id) { + exists.clone() + } else { + Gc::new(ByteCodeLambda::from_serialized(&mut serializer, v)) + }, + ) + }) + .collect(), + spans: thread + .function_interner + .spans + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + }; + + println!("Initialized function interner"); + + // Patch over the values in the final heap! + + for (key, value) in serializer.values_to_fill_in { + if let Some(cycled) = serializer.fake_heap.get(key) { + match cycled { + SerializedHeapRef::Serialized(_) => todo!(), + // Patch over the cycle + SerializedHeapRef::Closed(c) => { + value.set(c.get()); + } + } + } else { + todo!() + } + } + + println!("Patched heap"); + + drop(heap_guard); + + println!("Finished."); + + // New thread! It will result in a run time error if the function references globals that cannot be shared + // between threads. This is a bit of an unfortunate occurrence - we probably _should_ just have the engine share + // as much as possible between threads. + // let mut thread = SteelThread { + // global_env, + // sources, + // stack: Vec::with_capacity(64), + + // #[cfg(feature = "dynamic")] + // profiler: OpCodeOccurenceProfiler::new(), + + // function_interner, + // heap, + // runtime_options: thread.runtime_options, + // current_frame: StackFrame::main(), + // stack_frames: Vec::with_capacity(32), + // constant_map, + // interrupted: Default::default(), + // synchronizer: Synchronizer::new(), + // thread_local_storage: Vec::new(), + // // TODO: Fix this + // compiler: todo!(), + // id: EngineId::new(), + // safepoints_enabled: false, + // }; + + // #[cfg(feature = "profiling")] + // log::info!(target: "threads", "Time taken to spawn thread: {:?}", now.elapsed()); + + // Call the function! + // thread + // .call_function( + // thread.constant_map.clone(), + // SteelVal::Closure(Gc::new(closure)), + // Vec::new(), + // ) + // .map_err(|e| e.to_string()) + + // return ThreadHandle { + // handle: Some(handle), + // thread_state_manager: ThreadStateController::default(), + // } + // .into_steelval(); + + Ok(SteelVal::Void) +} pub struct SteelReceiver { receiver: crossbeam_channel::Receiver, @@ -867,8 +889,14 @@ impl Custom for std::thread::ThreadId { } } +#[derive(Clone)] pub struct ThreadLocalStorage(usize); -impl crate::rvals::Custom for ThreadLocalStorage {} +impl crate::rvals::Custom for ThreadLocalStorage { + fn into_serializable_steelval(&mut self) -> Option { + // TODO: This probably should have a different implementation? + Some(SerializableSteelVal::Custom(Box::new(self.clone()))) + } +} /// Creates a thread local storage slot. These slots are static, and will _not_ be reclaimed. /// @@ -958,20 +986,22 @@ pub fn threading_module() -> BuiltInModule { |channel: &std::sync::mpsc::Sender, val: SteelVal| -> Result<()> { - let mut map = HashMap::new(); - let mut visited = HashSet::new(); + // let mut map = HashMap::new(); + // let mut visited = HashSet::new(); + + todo!() // TODO: Handle this here somehow, we don't want to use an empty map - let serializable = - crate::rvals::into_serializable_value(val, &mut map, &mut visited)?; + // let serializable = + // crate::rvals::into_serializable_value(val, &mut map, &mut visited)?; - if !map.is_empty() { - stop!(Generic => "Unable to send mutable variable over a channel"); - } + // if !map.is_empty() { + // stop!(Generic => "Unable to send mutable variable over a channel"); + // } - channel - .send(serializable) - .map_err(|e| SteelErr::new(ErrorKind::Generic, e.to_string())) + // channel + // .send(serializable) + // .map_err(|e| SteelErr::new(ErrorKind::Generic, e.to_string())) }, ) // TODO: These need to be fucntions that take the context diff --git a/crates/steel-core/src/values/functions.rs b/crates/steel-core/src/values/functions.rs index 172c30af0..040dbb10f 100644 --- a/crates/steel-core/src/values/functions.rs +++ b/crates/steel-core/src/values/functions.rs @@ -46,11 +46,17 @@ use super::{ // Keep track of this metadata table for getting the docs associated // with a given function? + +#[derive(Clone)] pub struct LambdaMetadataTable { fn_ptr_table: HashMap, } -impl Custom for LambdaMetadataTable {} +impl Custom for LambdaMetadataTable { + fn into_serializable_steelval(&mut self) -> Option { + Some(SerializableSteelVal::Custom(Box::new(self.clone()))) + } +} impl LambdaMetadataTable { pub fn new() -> Self { diff --git a/crates/steel-core/src/values/lazy_stream.rs b/crates/steel-core/src/values/lazy_stream.rs index 4c1eea332..ce48a5bd2 100644 --- a/crates/steel-core/src/values/lazy_stream.rs +++ b/crates/steel-core/src/values/lazy_stream.rs @@ -1,4 +1,4 @@ -use crate::rvals::SteelVal; +use crate::rvals::{SerializableSteelVal, SteelVal}; #[derive(Clone)] pub struct LazyStream { @@ -7,6 +7,12 @@ pub struct LazyStream { pub empty_stream: bool, } +pub struct SerializableStream { + pub initial_value: SerializableSteelVal, + pub stream_thunk: SerializableSteelVal, + pub empty_stream: bool, +} + impl LazyStream { // Perhaps do some error checking here in order to determine // if the arguments passed are actually valid diff --git a/crates/steel-core/src/values/structs.rs b/crates/steel-core/src/values/structs.rs index 156063912..51b4147f1 100644 --- a/crates/steel-core/src/values/structs.rs +++ b/crates/steel-core/src/values/structs.rs @@ -12,7 +12,7 @@ use crate::parser::interner::InternedString; use crate::rerrs::ErrorKind; use crate::rvals::{ from_serializable_value, into_serializable_value, Custom, HeapSerializer, SerializableSteelVal, - SerializedHeapRef, SteelHashMap, + SerializationContext, SerializedHeapRef, SteelHashMap, }; use crate::rvals::{FromSteelVal, IntoSteelVal}; use crate::steel_vm::register_fn::RegisterFn; @@ -665,8 +665,7 @@ impl VTable { } pub(crate) fn sendable_entries( - serializer: &mut std::collections::HashMap, - visited: &mut std::collections::HashSet, + ctx: &mut SerializationContext, ) -> Result> { VTABLE.with(|x| { x.borrow() @@ -683,8 +682,8 @@ impl VTable { .iter() .map(|(key, value)| { Ok(( - into_serializable_value(key.clone(), serializer, visited)?, - into_serializable_value(value.clone(), serializer, visited)?, + into_serializable_value(key.clone(), ctx)?, + into_serializable_value(value.clone(), ctx)?, )) }) .collect::>>()?, From abb1b9123afe62a2bbd168a094ae63eff6d08fb7 Mon Sep 17 00:00:00 2001 From: Matt Paras Date: Thu, 6 Nov 2025 18:50:23 -0800 Subject: [PATCH 2/2] checkpoint --- crates/steel-core/src/rvals.rs | 90 +++++++++++--- crates/steel-core/src/steel_vm/vm/threads.rs | 124 +++++++++---------- crates/steel-core/src/values/closed.rs | 10 +- crates/steel-core/src/values/functions.rs | 1 + crates/steel-core/src/values/lazy_stream.rs | 1 + crates/steel-core/src/values/structs.rs | 1 + 6 files changed, 146 insertions(+), 81 deletions(-) diff --git a/crates/steel-core/src/rvals.rs b/crates/steel-core/src/rvals.rs index 3e2fd5c8b..093c061c6 100644 --- a/crates/steel-core/src/rvals.rs +++ b/crates/steel-core/src/rvals.rs @@ -179,6 +179,14 @@ pub trait Custom: private::Sealed { None } + fn from_serializable_steelval( + &mut self, + ctx: &mut HeapSerializer, + spec: SerializedNativeStructSpec, + ) -> Option { + None + } + fn as_iterator(&self) -> Option>> { None } @@ -911,15 +919,13 @@ impl From for SteelVal { } } -// TODO: -// This needs to be a method on the runtime: in order to properly support -// threads -// Tracking issue here: https://github.com/mattwparas/steel/issues/98 +pub struct SerializedNativeStructSpec {} // Values which can be sent to another thread. // If it cannot be sent to another thread, then we'll error out on conversion. // TODO: Add boxed dyn functions to this. // #[derive(PartialEq)] + pub enum SerializableSteelVal { Closure(crate::values::functions::SerializedLambda), BoolV(bool), @@ -949,6 +955,40 @@ pub enum SerializableSteelVal { NativeRef(NativeRefSpec), } +impl std::fmt::Debug for SerializableSteelVal { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Custom(c) => write!(f, "#"), + SerializableSteelVal::Closure(serialized_lambda) => { + write!(f, "{:?}", serialized_lambda) + } + SerializableSteelVal::BoolV(x) => write!(f, "{}", x), + SerializableSteelVal::NumV(x) => write!(f, "{}", x), + SerializableSteelVal::IntV(x) => write!(f, "{}", x), + SerializableSteelVal::CharV(x) => write!(f, "{}", x), + SerializableSteelVal::Void => write!(f, "SteelVal::Void"), + SerializableSteelVal::StringV(x) => write!(f, "{}", x), + SerializableSteelVal::FuncV(x) => write!(f, "{:?}", x), + SerializableSteelVal::MutFunc(x) => write!(f, "{:?}", x), + SerializableSteelVal::HashMapV(x) => write!(f, "{:?}", x), + SerializableSteelVal::HashSet(x) => write!(f, "{:?}", x), + SerializableSteelVal::ListV(x) => write!(f, "{:?}", x), + SerializableSteelVal::Pair(x) => write!(f, "{:?}", x), + SerializableSteelVal::VectorV(x) => write!(f, "{:?}", x), + SerializableSteelVal::ByteVectorV(items) => write!(f, "{:?}", items), + SerializableSteelVal::BoxedDynFunction(x) => write!(f, "#"), + SerializableSteelVal::BuiltIn(x) => write!(f, "{:?}", x), + SerializableSteelVal::SymbolV(x) => write!(f, "{:?}", x), + SerializableSteelVal::CustomStruct(x) => write!(f, "{:?}", x), + SerializableSteelVal::HeapAllocated(x) => write!(f, "{:?}", x), + SerializableSteelVal::Port(x) => write!(f, "#"), + SerializableSteelVal::Rational(x) => write!(f, "{:?}", x), + SerializableSteelVal::Stream(x) => write!(f, "{:?}", x), + SerializableSteelVal::NativeRef(x) => write!(f, "{:?}", x), + } + } +} + /* Serialize via known modules. If the value is a native one, then it should get replaced with a value that is something like SerializableSteelVal::NativeRef(NativeRefSpec) where NativeRefSpec is like: @@ -961,6 +1001,7 @@ struct NativeRefSpec { And then deserializing is just grabbing that back from the root. */ +#[derive(Debug)] pub struct NativeRefSpec { pub module: String, pub key: String, @@ -980,6 +1021,8 @@ pub struct HeapSerializer<'a> { // Cache the functions that get built pub built_functions: &'a mut std::collections::HashMap>, + + pub modules: ModuleContainer, } // Once crossed over the line, convert BACK into a SteelVal @@ -1077,23 +1120,35 @@ pub fn from_serializable_value(ctx: &mut HeapSerializer, val: SerializableSteelV if let Some(mut guard) = ctx.fake_heap.get_mut(&v) { match &mut guard { SerializedHeapRef::Serialized(value) => { + println!("Visiting: {} -> {:?}", v, value); + let value = std::mem::take(value); if let Some(value) = value { - let _ = from_serializable_value(ctx, value); + let value = from_serializable_value(ctx, value); - todo!() - // let allocation = ctx.heap.allocate_without_collection(value); + let allocation = ctx.heap.allocate_without_collection(value); - // ctx.fake_heap - // .insert(v, SerializedHeapRef::Closed(allocation.clone())); + ctx.fake_heap + .insert(v, SerializedHeapRef::Closed(allocation.clone())); - // SteelVal::HeapAllocated(allocation) + println!("patching: {}", v); + + SteelVal::HeapAllocated(allocation) } else { // println!("If we're getting here - it means the value from the heap has already // been converting. if so, we should do something..."); - todo!() + // todo!() + + match ctx.fake_heap.get(&v).unwrap() { + SerializedHeapRef::Serialized(serializable_steel_val) => { + panic!("Found a cycle: {}", v); + } + SerializedHeapRef::Closed(heap_ref) => { + SteelVal::HeapAllocated(heap_ref.clone()) + } + } // let fake_allocation = // ctx.heap.allocate_without_collection(SteelVal::Void); @@ -1138,8 +1193,14 @@ pub fn from_serializable_value(ctx: &mut HeapSerializer, val: SerializableSteelV empty_stream: value.empty_stream, })), - SerializableSteelVal::NativeRef(_) => { - todo!("Implement native ref spec deserialization!") + SerializableSteelVal::NativeRef(s) => { + let module_map = ctx.modules.inner(); + + if let Some(m) = module_map.get(s.module.as_str()) { + return m.get(s.key); + } + + panic!("Unable to find value in module map: {:#?}", s); } } } @@ -1172,8 +1233,7 @@ pub fn into_serializable_value( SteelVal::CharV(c) => Ok(SerializableSteelVal::CharV(c)), SteelVal::Void => Ok(SerializableSteelVal::Void), SteelVal::StringV(s) => Ok(SerializableSteelVal::StringV(s.to_string())), - // SteelVal::FuncV(f) => Ok(SerializableSteelVal::FuncV(f)), - SteelVal::FuncV(f) => Ok(SerializableSteelVal::NativeRef( + SteelVal::FuncV(_) => Ok(SerializableSteelVal::NativeRef( // TODO: Native ref spec for anything that is native // and truly can't be serialized between runtimes, such as native // functions. diff --git a/crates/steel-core/src/steel_vm/vm/threads.rs b/crates/steel-core/src/steel_vm/vm/threads.rs index 9c8c6b4a5..5f9065f79 100644 --- a/crates/steel-core/src/steel_vm/vm/threads.rs +++ b/crates/steel-core/src/steel_vm/vm/threads.rs @@ -210,7 +210,7 @@ struct MovableThread { constants: Vec, global_env: Vec, function_interner: MovableFunctionInterner, - runtime_options: RunTimeOptions, + _runtime_options: RunTimeOptions, } struct MovableFunctionInterner { @@ -258,7 +258,7 @@ pub(crate) fn create_native_ref(ctx: &ModuleContainer, v: SteelVal) -> Option Result { +fn serialize_thread_impl(ctx: &mut VmCore, _args: &[SteelVal]) -> Result { // use crate::rvals::SerializableSteelVal; #[cfg(feature = "profiling")] @@ -309,7 +309,7 @@ fn serialize_thread_impl(ctx: &mut VmCore, args: &[SteelVal]) -> Result Result Result Result Result Result BuiltInModule { }) .register_fn( "channel->send", - |channel: &std::sync::mpsc::Sender, - val: SteelVal| + |_channel: &std::sync::mpsc::Sender, + _val: SteelVal| -> Result<()> { // let mut map = HashMap::new(); // let mut visited = HashSet::new(); @@ -994,62 +995,55 @@ pub fn threading_module() -> BuiltInModule { }, ) // TODO: These need to be fucntions that take the context - .register_fn("channel->recv", |channel: &SReceiver| -> Result { - let receiver = channel - .receiver - .as_ref() - .expect("Channel should not be dropped here!"); - - let value = receiver - .recv() - .map_err(|e| SteelErr::new(ErrorKind::Generic, e.to_string()))?; - - let mut heap = Heap::new_empty(); - let mut fake_heap = HashMap::new(); - let mut patcher = HashMap::new(); - let mut built_functions = HashMap::new(); - let mut serializer = HeapSerializer { - heap: &mut heap, - fake_heap: &mut fake_heap, - values_to_fill_in: &mut patcher, - built_functions: &mut built_functions, - }; - - let value = crate::rvals::from_serializable_value(&mut serializer, value); - - Ok(value) - }) - .register_fn( - "channel->try-recv", - |channel: &SReceiver| -> Result> { - let receiver = channel - .receiver - .as_ref() - .expect("Channel should not be dropped here!"); - - let value = receiver.try_recv(); - - let mut heap = Heap::new_empty(); - let mut fake_heap = HashMap::new(); - let mut patcher = HashMap::new(); - let mut built_functions = HashMap::new(); - let mut serializer = HeapSerializer { - heap: &mut heap, - fake_heap: &mut fake_heap, - values_to_fill_in: &mut patcher, - built_functions: &mut built_functions, - }; - - match value { - Ok(v) => Ok(Some(crate::rvals::from_serializable_value( - &mut serializer, - v, - ))), - Err(std::sync::mpsc::TryRecvError::Empty) => Ok(None), - Err(e) => Err(SteelErr::new(ErrorKind::Generic, e.to_string())), - } - }, - ) + // .register_fn("channel->recv", |channel: &SReceiver| -> Result { + // let receiver = channel + // .receiver + // .as_ref() + // .expect("Channel should not be dropped here!"); + // let value = receiver + // .recv() + // .map_err(|e| SteelErr::new(ErrorKind::Generic, e.to_string()))?; + // let mut heap = Heap::new_empty(); + // let mut fake_heap = HashMap::new(); + // let mut patcher = HashMap::new(); + // let mut built_functions = HashMap::new(); + // let mut serializer = HeapSerializer { + // heap: &mut heap, + // fake_heap: &mut fake_heap, + // values_to_fill_in: &mut patcher, + // built_functions: &mut built_functions, + // }; + // let value = crate::rvals::from_serializable_value(&mut serializer, value); + // Ok(value) + // }) + // .register_fn( + // "channel->try-recv", + // |channel: &SReceiver| -> Result> { + // let receiver = channel + // .receiver + // .as_ref() + // .expect("Channel should not be dropped here!"); + // let value = receiver.try_recv(); + // let mut heap = Heap::new_empty(); + // let mut fake_heap = HashMap::new(); + // let mut patcher = HashMap::new(); + // let mut built_functions = HashMap::new(); + // let mut serializer = HeapSerializer { + // heap: &mut heap, + // fake_heap: &mut fake_heap, + // values_to_fill_in: &mut patcher, + // built_functions: &mut built_functions, + // }; + // match value { + // Ok(v) => Ok(Some(crate::rvals::from_serializable_value( + // &mut serializer, + // v, + // ))), + // Err(std::sync::mpsc::TryRecvError::Empty) => Ok(None), + // Err(e) => Err(SteelErr::new(ErrorKind::Generic, e.to_string())), + // } + // }, + // ) .register_fn("thread::current/id", || std::thread::current().id()) .register_fn("thread/available-parallelism", || { std::thread::available_parallelism().map(|x| x.get()).ok() diff --git a/crates/steel-core/src/values/closed.rs b/crates/steel-core/src/values/closed.rs index 1f3b3d851..114d6adc0 100644 --- a/crates/steel-core/src/values/closed.rs +++ b/crates/steel-core/src/values/closed.rs @@ -697,7 +697,11 @@ impl WillExecutor { } } -impl Custom for WillExecutor {} +impl Custom for WillExecutor { + fn into_serializable_steelval(&mut self) -> Option { + Some(crate::rvals::SerializableSteelVal::Void) + } +} #[cfg(feature = "sync")] #[steel_derive::function(name = "make-will-executor")] @@ -1606,6 +1610,10 @@ impl Heap { ); } + pub fn allocate_without_collection(&mut self, value: SteelVal) -> HeapRef { + self.memory_free_list.allocate(value) + } + // Clean up the values? pub fn allocate<'a>( &mut self, diff --git a/crates/steel-core/src/values/functions.rs b/crates/steel-core/src/values/functions.rs index 282989a72..f45650f78 100644 --- a/crates/steel-core/src/values/functions.rs +++ b/crates/steel-core/src/values/functions.rs @@ -158,6 +158,7 @@ impl std::hash::Hash for ByteCodeLambda { // Can this be moved across threads? What does it cost to execute a closure in another thread? // Engine instances be deep cloned? +#[derive(Debug)] pub struct SerializedLambda { pub id: u32, pub body_exp: Vec, diff --git a/crates/steel-core/src/values/lazy_stream.rs b/crates/steel-core/src/values/lazy_stream.rs index ce48a5bd2..d8c07da86 100644 --- a/crates/steel-core/src/values/lazy_stream.rs +++ b/crates/steel-core/src/values/lazy_stream.rs @@ -7,6 +7,7 @@ pub struct LazyStream { pub empty_stream: bool, } +#[derive(Debug)] pub struct SerializableStream { pub initial_value: SerializableSteelVal, pub stream_thunk: SerializableSteelVal, diff --git a/crates/steel-core/src/values/structs.rs b/crates/steel-core/src/values/structs.rs index 84e8640fd..3bd317f6c 100644 --- a/crates/steel-core/src/values/structs.rs +++ b/crates/steel-core/src/values/structs.rs @@ -124,6 +124,7 @@ impl StructTypeDescriptor { } } +#[derive(Debug)] pub struct SerializableUserDefinedStruct { pub(crate) fields: Vec,