diff --git a/crates/steel-core/src/compiler/modules.rs b/crates/steel-core/src/compiler/modules.rs index e790e2268..54cf184a6 100644 --- a/crates/steel-core/src/compiler/modules.rs +++ b/crates/steel-core/src/compiler/modules.rs @@ -1047,6 +1047,10 @@ impl CompiledModule { } } + pub fn dependent_modules(&self) -> &[PathBuf] { + &self.downstream + } + // TODO: Should cache this pub fn prefix(&self) -> CompactString { self.cached_prefix.clone() diff --git a/crates/steel-core/src/env.rs b/crates/steel-core/src/env.rs index bafb09e13..1a67cad80 100644 --- a/crates/steel-core/src/env.rs +++ b/crates/steel-core/src/env.rs @@ -44,7 +44,7 @@ pub struct Env { #[cfg(not(feature = "sync"))] pub(crate) bindings_vec: Vec, #[cfg(feature = "sync")] - bindings: SharedVectorWrapper, + pub(crate) bindings: SharedVectorWrapper, } #[cfg(feature = "sync")] diff --git a/crates/steel-core/src/steel_vm/engine.rs b/crates/steel-core/src/steel_vm/engine.rs index d4ab5c2a2..118ee3d67 100644 --- a/crates/steel-core/src/steel_vm/engine.rs +++ b/crates/steel-core/src/steel_vm/engine.rs @@ -56,6 +56,7 @@ use crate::{ values::{ closed::GlobalSlotRecycler, functions::{BoxedDynFunction, ByteCodeLambda}, + reachable::GlobalSlotReacher, }, SteelErr, }; @@ -2749,6 +2750,51 @@ fn test_raw_engine() { } #[test] +fn namespace_querying() { + let mut engine = Engine::new(); + + let module = PathBuf::from("#%private/steel/contract"); + + let guard = engine.virtual_machine.compiler.read(); + let compiled_module = guard.module_manager.modules().get(&module).unwrap(); + let module_name: InternedString = format!("__module-{}", compiled_module.prefix()).into(); + let index = guard.symbol_map.get(&module_name).unwrap(); + let value = engine + .virtual_machine + .global_env + .repl_maybe_lookup_idx(index) + .unwrap(); + + if let SteelVal::HashMapV(h) = value { + let mut roots = Vec::new(); + + for (key, value) in h.iter() { + roots.push(value.clone()); + } + + let reachable_globals = GlobalSlotReacher::find_reachable( + engine.virtual_machine.global_env.roots(), + &mut roots, + ); + + // Module -> Set + // + // GlobalIndex (old) -> GlobalIndex (new) + // + // Patch old values into new environment + // + // Global Slot Renamer + // + // Iterates over everything, renames the indexes to the new spots. + + println!("{:#?}", reachable_globals); + + // None + } else { + // return None; + } +} + fn test_ctx_func() { let mut engine = Engine::new(); diff --git a/crates/steel-core/src/steel_vm/primitives.rs b/crates/steel-core/src/steel_vm/primitives.rs index ccc58749d..0300ecec3 100644 --- a/crates/steel-core/src/steel_vm/primitives.rs +++ b/crates/steel-core/src/steel_vm/primitives.rs @@ -59,7 +59,8 @@ use crate::{ steel_vm::{ builtin::{get_function_metadata, get_function_name, BuiltInFunctionType}, vm::{ - threads::threading_module, GET_MODULE_CONTEXT_DEFINITION, + threads::threading_module, EVAL_WITH_NAMESPACE_DEFINITION, + GET_MODULE_CONTEXT_DEFINITION, MAKE_NAMESPACE_DEFINITION, NAMESPACE_REQUIRE_DEFINITION, POP_MODULE_CONTEXT_DEFINITION, PUSH_MODULE_CONTEXT_DEFINITION, }, }, @@ -2344,7 +2345,11 @@ fn meta_module() -> BuiltInModule { module .register_native_fn_definition(WILL_EXECUTE_DEFINITION) .register_native_fn_definition(WILL_REGISTER_DEFINITION) - .register_native_fn_definition(MAKE_WILL_EXECUTOR_DEFINITION); + .register_native_fn_definition(MAKE_WILL_EXECUTOR_DEFINITION) + // EVAL WITH NAMESPACE STUFF + .register_native_fn_definition(MAKE_NAMESPACE_DEFINITION) + .register_native_fn_definition(NAMESPACE_REQUIRE_DEFINITION) + .register_native_fn_definition(EVAL_WITH_NAMESPACE_DEFINITION); #[cfg(not(feature = "dylibs"))] module.register_native_fn_definition(super::engine::LOAD_MODULE_NOOP_DEFINITION); diff --git a/crates/steel-core/src/steel_vm/vm.rs b/crates/steel-core/src/steel_vm/vm.rs index 16b3e4628..ca2343342 100644 --- a/crates/steel-core/src/steel_vm/vm.rs +++ b/crates/steel-core/src/steel_vm/vm.rs @@ -1,3 +1,4 @@ +use crate::compiler::code_gen::fresh_function_id; use crate::compiler::compiler::Compiler; use crate::compiler::passes::VisitorMutRefUnit; use crate::core::instructions::{pretty_print_dense_instructions, u24}; @@ -36,6 +37,7 @@ use crate::values::functions::CaptureVec; use crate::values::functions::RootedInstructions; use crate::values::functions::SerializedLambda; use crate::values::lists::List; +use crate::values::reachable::GlobalSlotReacher; use crate::values::structs::UserDefinedStruct; use crate::values::transducers::Reducer; use crate::{ @@ -165,6 +167,295 @@ impl DehydratedStackTrace { pub struct StackFrameAttachments { pub(crate) handler: Option, weak_continuation_mark: Option, + namespace: Option>>, +} + +// Attempt to... figure out a namespace. +// I'm guessing this isn't gonna work great but +// its worth a try. +#[derive(Clone)] +pub struct SteelNamespace { + env: Env, + compiler: alloc::sync::Arc>, +} + +// Represents a base level environment +// More or less the kernel environment + +// (define foo bar) +// (define ns (make-namespace)) +// (require "foo.scm") ;; instantiates a bunch of stuff +// +// (provide-module ns "foo.scm") ;; (require "foo.scm") +pub struct Namespace { + namespace: Arc>, +} + +impl crate::rvals::Custom for Namespace {} + +impl SteelNamespace { + pub fn new(ctx: &mut VmCore) -> Self { + let guard = ctx.thread.compiler.read(); + let sources = guard.sources.clone(); + let builtin_modules = guard.builtin_modules.clone(); + + Self { + env: Env::root(), + compiler: alloc::sync::Arc::new(RwLock::new(Compiler::default_with_kernel( + sources, + builtin_modules, + ))), + } + } + + pub fn require_module(&mut self, ctx: &mut VmCore, module: PathBuf) -> Option<()> { + let guard = ctx.thread.compiler.read(); + let compiled_module = guard.module_manager.modules().get(&module).or_else(|| { + guard + .module_manager + .modules() + .get(&std::fs::canonicalize(&module).ok()?) + })?; + + println!("Requiring module: {:?}", module); + + // Modules: + + let dependent_modules = compiled_module.dependent_modules(); + + let mut mapping = HashMap::new(); + + let primitives_to_map = ["%proto-hash-get%".into()]; + + for primitive in primitives_to_map { + let index = guard.symbol_map.get(&primitive).ok()?; + let value = ctx.thread.global_env.repl_maybe_lookup_idx(index)?; + + // Register this module as well: + let new_index = self.compiler.write().symbol_map.add(&primitive); + SharedVectorWrapper::repl_define_idx(&mut self.env.bindings, new_index, value.clone()); + } + + for compiled_module in std::iter::once(compiled_module).chain( + dependent_modules + .iter() + .filter_map(|x| guard.module_manager.modules().get(x)), + ) { + // Add the module to the namespace + self.compiler.write().module_manager.modules_mut().insert( + compiled_module.name().to_path_buf(), + compiled_module.clone(), + ); + + let module_name: InternedString = + format!("__module-{}", compiled_module.prefix()).into(); + let index = guard.symbol_map.get(&module_name).ok()?; + let value = ctx.thread.global_env.repl_maybe_lookup_idx(index)?; + + println!("Binding: {}", module_name); + println!("{}", value); + + // Register this module as well: + let new_index = self.compiler.write().symbol_map.add(&module_name); + + SharedVectorWrapper::repl_define_idx(&mut self.env.bindings, new_index, value.clone()); + + if let SteelVal::HashMapV(h) = &value { + let mut roots = Vec::new(); + + for (_, value) in h.iter() { + roots.push(value.clone()); + } + + let reachable_globals = + GlobalSlotReacher::find_reachable(ctx.thread.global_env.roots(), &mut roots); + + for index in &reachable_globals { + let name = guard.symbol_map.values()[*index]; + let idx = self.compiler.write().symbol_map.add(&name); + mapping.insert(*index, idx); + } + + for index in reachable_globals { + let mut global = ctx.thread.global_env.repl_maybe_lookup_idx(index).unwrap(); + + rewrite_indices(&mut ctx.thread.function_interner, &mapping, &mut global); + + let idx = mapping.get(&index).unwrap(); + + SharedVectorWrapper::repl_define_idx(&mut self.env.bindings, *idx, global); + } + + for (name, value) in h.iter() { + if let SteelVal::SymbolV(s) = name { + // This could fail if the module was only partially required, and then it doesn't + // exist in the global namespace. If that is the case, we should just make a new + // thing to fetch, and then visit it + // let name = guard + // .symbol_map + // .get(&InternedString::from_str(s.as_str())) + // .unwrap(); + + println!("Trying to define: {}", s); + let idx = self + .compiler + .write() + .symbol_map + .add(&InternedString::from_str(s.as_str())); + + let mut global = value.clone(); + rewrite_indices(&mut ctx.thread.function_interner, &mapping, &mut global); + + SharedVectorWrapper::repl_define_idx(&mut self.env.bindings, idx, global); + } + } + } + } + + None + } +} + +fn rewrite_indices( + ctx: &mut FunctionInterner, + mapping: &HashMap, + global: &mut SteelVal, +) { + let mut stack: Vec = Vec::new(); + + // This is the global that is going to be used + if let SteelVal::Closure(c) = &*global { + let mut c = c.unwrap(); + c.id = fresh_function_id() as _; + + rewrite_indices_closure(ctx, mapping, &mut c, &mut stack); + *global = SteelVal::Closure(Gc::new(c)); + } + + while let Some(mut c) = stack.pop() { + println!("Rewriting closure: {}", c.id); + rewrite_indices_closure(ctx, mapping, &mut c, &mut stack); + } +} + +fn rewrite_indices_closure( + ctx: &mut FunctionInterner, + mapping: &HashMap, + c: &mut ByteCodeLambda, + stack: &mut Vec, +) { + let mut new_body = Vec::with_capacity(c.body_exp().len()); + + let mut closures_to_rewrite = Vec::new(); + + for (idx, instr) in c.body_exp().iter().enumerate() { + let mut instr = *instr; + + // TODO: Rewrite the closures these reference as well. + // If there is an SCLOSURE or something, then a new closure + // will get created (or maybe, was already created) that pointed + // to the old one? + match instr.op_code { + // If this instruction touches this global variable, + // then we want to mark it as possibly referenced here. + OpCode::CALLGLOBAL + | OpCode::CALLPRIMITIVE + | OpCode::PUSH + | OpCode::CALLGLOBALTAIL + | OpCode::CALLGLOBALNOARITY + | OpCode::CALLGLOBALTAILNOARITY => { + instr.payload_size = + u24::from_usize(*mapping.get(&instr.payload_size.to_usize()).unwrap()); + } + + // TODO: Find the ip of the closure, and then go allocate a _new_ closure + // for this one, since the values are going to now be rewritten. + OpCode::NEWSCLOSURE => { + // closure IP + let closure_id = c.body_exp().get(idx + 2).unwrap().payload_size.to_u32(); + + if let Some(value) = ctx.closure_interner.get(&closure_id) { + println!( + "Found closure that has already been put into the cache: {}", + closure_id + ); + let mut value = value.clone(); + value.id = fresh_function_id() as _; + + closures_to_rewrite.push((idx + 2, value.id)); + + stack.push(value.clone()); + } + } + OpCode::PUREFUNC => { + let closure_id = c.body_exp().get(idx + 2).unwrap().payload_size.to_u32(); + closures_to_rewrite.push((idx + 2, fresh_function_id() as _)); + + if let Some(value) = ctx.pure_function_interner.get(&closure_id) { + println!( + "Found pure function that has already been put into the cache: {}", + closure_id + ); + // let mut value = value.clone(); + // value.id = fresh_function_id() as _; + + // closures_to_rewrite.push((idx + 2, value.id)); + + // stack.push(value.clone()); + } + } + _ => {} + } + + new_body.push(instr); + } + + for (idx, id) in closures_to_rewrite { + new_body[idx].payload_size = u24::from_u32(id); + } + + c.body_exp = Arc::from(new_body); + ctx.closure_interner.insert(c.id, c.clone()); +} + +#[steel_derive::context(name = "make-namespace", arity = "Exact(0)")] +fn make_namespace(ctx: &mut VmCore, args: &[SteelVal]) -> Option> { + Some( + Namespace { + namespace: Arc::new(Mutex::new(SteelNamespace::new(ctx))), + } + .into_steelval(), + ) +} + +#[steel_derive::context(name = "namespace-require", arity = "Exact(2)")] +fn namespace_require(ctx: &mut VmCore, args: &[SteelVal]) -> Option> { + fn namespace_require_impl(ctx: &mut VmCore, args: &[SteelVal]) -> Result { + let namespace = Namespace::as_mut_ref(&args[0])?; + let path = String::from_steelval(&args[1])?; + + namespace + .namespace + .lock() + .unwrap() + .require_module(ctx, PathBuf::from(path)); + + Ok(SteelVal::Void) + } + + Some(namespace_require_impl(ctx, args)) +} + +impl std::fmt::Debug for Namespace { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "#") + } +} + +impl std::fmt::Debug for SteelNamespace { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "#") + } } // This should be the go to thing for handling basically everything we need @@ -174,13 +465,9 @@ pub struct StackFrameAttachments { #[derive(Debug, Clone)] pub struct StackFrame { sp: u32, - pub(crate) function: Gc, - ip: u32, - instructions: RootedInstructions, - pub(crate) attachments: Option>, } @@ -247,6 +534,7 @@ impl StackFrame { self.attachments = Some(Box::new(StackFrameAttachments { handler: None, weak_continuation_mark: Some(WeakContinuation::from_strong(&continuation_mark)), + namespace: None, })) } } @@ -283,6 +571,7 @@ impl StackFrame { self.attachments = Some(Box::new(StackFrameAttachments { handler: None, weak_continuation_mark: Some(WeakContinuation::from_strong(continuation)), + namespace: None, })) } } @@ -300,6 +589,7 @@ impl StackFrame { self.attachments = Some(Box::new(StackFrameAttachments { handler: Some(handler), weak_continuation_mark: None, + namespace: None, })) } } @@ -1245,9 +1535,11 @@ impl SteelThread { return Err(e); } else { - for frame in &vm_instance.thread.stack_frames { - Continuation::close_marks(&vm_instance, frame); - } + vm_instance.close_all_continuation_marks_by_value(); + + // for frame in &vm_instance.thread.stack_frames { + // Continuation::close_marks(&mut vm_instance, frame); + // } // Clean up self.stack.clear(); @@ -1344,26 +1636,70 @@ impl ContinuationMark { impl Continuation { #[inline(always)] - pub fn close_marks(ctx: &VmCore<'_>, stack_frame: &StackFrame) -> bool { - // if let Some(cont_mark) = stack_frame - // .weak_continuation_mark - // .as_ref() - // .and_then(|x| WeakShared::upgrade(&x.inner)) - // { - - if let Some(cont_mark) = stack_frame.attachments.as_ref().and_then(|x| { - x.weak_continuation_mark + pub fn close_marks(ctx: &mut VmCore<'_>, stack_frame: &StackFrame) -> bool { + if let Some(attachments) = stack_frame.attachments.as_ref() { + let mut ret = false; + + if let Some(cont_mark) = attachments + .weak_continuation_mark .as_ref() .and_then(|x| WeakShared::upgrade(&x.inner)) - }) { - cont_mark.write().close(ctx); + { + cont_mark.write().close(ctx); + ret = true; + } + + if let Some(ns) = attachments.namespace.as_ref() { + let mut guard = ns.lock().unwrap(); + std::mem::swap(&mut ctx.thread.global_env, &mut guard.env); + std::mem::swap(&mut ctx.thread.compiler, &mut guard.compiler); + } - return true; + return ret; } false } + pub fn close_marks_by_value(ctx: &mut VmCore<'_>, stack_frame: &StackFrame) -> bool { + if let Some(attachments) = stack_frame.attachments.as_ref() { + let mut ret = false; + + if let Some(cont_mark) = attachments + .weak_continuation_mark + .as_ref() + .and_then(|x| WeakShared::upgrade(&x.inner)) + { + cont_mark.write().close(ctx); + ret = true; + } + + // When we're here, I want to update the existing + // namespace in order to get things to behave. + if let Some(ns) = attachments.namespace.as_ref() { + let mut guard = ns.lock().unwrap(); + std::mem::swap(&mut ctx.thread.global_env, &mut guard.env); + std::mem::swap(&mut ctx.thread.compiler, &mut guard.compiler); + } + + return ret; + } + + false + + // if let Some(cont_mark) = stack_frame.attachments.as_ref().and_then(|x| { + // x.weak_continuation_mark + // .as_ref() + // .and_then(|x| WeakShared::upgrade(&x.inner)) + // }) { + // cont_mark.write().close(ctx); + + // return true; + // } + + // false + } + pub fn set_state_from_continuation(ctx: &mut VmCore<'_>, this: Self) { // Check if this is an open let maybe_open_mark = (*this.inner.read()).clone().into_open_mark(); @@ -1694,7 +2030,7 @@ impl<'a> VmCore<'a> { #[cfg(feature = "sync")] pub fn steel_function_to_rust_function( - &self, + &mut self, func: SteelVal, ) -> Box Result + Send + Sync + 'static> { let thread = self.make_thread(); @@ -1710,7 +2046,7 @@ impl<'a> VmCore<'a> { #[cfg(feature = "sync")] pub(crate) fn steel_function_to_arc_rust_function( - &self, + &mut self, func: SteelVal, ) -> Arc Result + Send + Sync + 'static> { let thread = self.make_thread(); @@ -1729,7 +2065,7 @@ impl<'a> VmCore<'a> { // TODO: Add this thread to the parent VM thread handler -> this is necessary // for safepoints to work correctly #[cfg(feature = "sync")] - pub fn make_thread(&self) -> Arc> { + pub fn make_thread(&mut self) -> Arc> { let mut thread = self.thread.clone(); let controller = ThreadStateController::default(); @@ -1764,10 +2100,13 @@ impl<'a> VmCore<'a> { handle: value.clone(), }); - for frame in &self.thread.stack_frames { - self.close_continuation_marks(frame); - } - self.close_continuation_marks(&self.thread.current_frame); + // for frame in &self.thread.stack_frames { + // self.close_continuation_marks(frame); + // } + + self.close_all_continuation_marks(); + + self.close_continuation_mark_current_frame(); forked_thread } @@ -3935,7 +4274,7 @@ impl<'a> VmCore<'a> { // stack will need to be instrumented with the point in time that we are at // with respect to the existing continuation. #[inline(always)] - fn close_continuation_marks(&self, last: &StackFrame) -> bool { + fn close_continuation_marks(&mut self, last: &StackFrame) -> bool { // TODO: @Matt - continuation marks should actually do something here // What we'd like: This marks the stack frame going out of scope. Since it is going out of scope, // the stack frame should check if there are marks here, specifying that we should grab @@ -3944,6 +4283,58 @@ impl<'a> VmCore<'a> { Continuation::close_marks(self, last) } + fn close_continuation_marks_by_value(&mut self, last: &StackFrame) -> bool { + // TODO: @Matt - continuation marks should actually do something here + // What we'd like: This marks the stack frame going out of scope. Since it is going out of scope, + // the stack frame should check if there are marks here, specifying that we should grab + // the values out of the existing frame, and "close" the open continuation. That way the continuation (if called) + // does not need to actually copy the entire frame eagerly, but rather can do so lazily. + Continuation::close_marks_by_value(self, &last) + } + + fn close_continuation_mark_current_frame(&mut self) -> bool { + if let Some(attachments) = self.thread.current_frame.attachments.as_ref() { + let mut ret = false; + + if let Some(cont_mark) = attachments + .weak_continuation_mark + .as_ref() + .and_then(|x| WeakShared::upgrade(&x.inner)) + { + cont_mark.write().close(self); + ret = true; + } + + if let Some(_ns) = attachments.namespace.as_ref() { + log::warn!("Found a namespace during continuation mark closing current frame"); + } + + return ret; + } + + false + } + + fn close_all_continuation_marks(&mut self) { + let frames = std::mem::take(&mut self.thread.stack_frames); + + for frame in &frames { + Continuation::close_marks(self, frame); + } + + self.thread.stack_frames = frames; + } + + fn close_all_continuation_marks_by_value(&mut self) { + let frames = std::mem::take(&mut self.thread.stack_frames); + + for frame in &frames { + Continuation::close_marks_by_value(self, frame); + } + + self.thread.stack_frames = frames; + } + #[inline(always)] fn handle_pop_pure_value(&mut self, value: SteelVal) -> Option> { // println!("calling pop pure value: {}", value); @@ -3963,7 +4354,7 @@ impl<'a> VmCore<'a> { let rollback_index = last.sp; - self.close_continuation_marks(&last); + self.close_continuation_marks_by_value(&last); // println!("Stack at pop: {:#?}", self.thread.stack); @@ -3995,7 +4386,7 @@ impl<'a> VmCore<'a> { let rollback_index = last .map(|x| { - self.close_continuation_marks(&x); + self.close_continuation_marks_by_value(&x); x.sp }) .unwrap_or(0); @@ -4028,7 +4419,7 @@ impl<'a> VmCore<'a> { let rollback_index = last.sp as _; - self.close_continuation_marks(&last); + self.close_continuation_marks_by_value(&last); let _ = self .thread @@ -4066,7 +4457,7 @@ impl<'a> VmCore<'a> { let rollback_index = last .map(|x| { - self.close_continuation_marks(&x); + self.close_continuation_marks_by_value(&x); x.sp }) .unwrap_or(0); @@ -4126,8 +4517,6 @@ impl<'a> VmCore<'a> { #[inline(always)] fn handle_call_global(&mut self, index: usize, payload_size: usize) -> Result<()> { - // TODO: Lazily fetch the function. Avoid cloning where relevant. - // Boxed functions probably _should_ be rooted in the modules? let func = self.thread.global_env.repl_lookup_idx(index); self.handle_global_function_call(func, payload_size) } @@ -5609,6 +5998,56 @@ pub fn call_cc(ctx: &mut VmCore, args: &[SteelVal]) -> Option> Some(Ok(SteelVal::ContinuationFunction(continuation))) } +fn eval_with_namespace_impl(ctx: &mut VmCore, args: &[SteelVal]) -> Result { + let mut expr = crate::parser::ast::TryFromSteelValVisitorForExprKind::root_quoted(&args[0])?; + + expr = steel_parser::parser::lower_macro_and_require_definitions(expr)?; + + let namespace = Namespace::as_mut_ref(&args[1])?; + + let maybe_path = ctx + .thread + .module_context + .last() + .map(|x| x.as_str().to_owned()) + .map(PathBuf::from); + + // TODO: Provide a path / context from which to do this? Or grab it implicitly? + let res = ctx + .thread + .compiler + .write() + .compile_executable_from_expressions_from_path(vec![expr], maybe_path); + + // Gc the sources, if possible + ctx.thread + .compiler + .write() + .gc_sources(ctx.thread.function_interner.spans.values()); + + match res { + Ok(program) => { + let result = program.build( + "eval-context".to_string(), + &mut namespace + .namespace + .lock() + .unwrap() + .compiler + .as_ref() + .write() + .symbol_map, + // &mut ctx.thread.compiler.write().symbol_map, + )?; + + eval_program_with_namespace(result, ctx, namespace.namespace.clone())?; + + Ok(SteelVal::Void) + } + Err(e) => Err(e), + } +} + fn eval_impl(ctx: &mut crate::steel_vm::vm::VmCore, args: &[SteelVal]) -> Result { // Can we have this... not lower? let mut expr = crate::parser::ast::TryFromSteelValVisitorForExprKind::root_quoted(&args[0])?; @@ -5650,6 +6089,184 @@ fn eval_impl(ctx: &mut crate::steel_vm::vm::VmCore, args: &[SteelVal]) -> Result } } +fn eval_program_with_namespace( + program: crate::compiler::program::Executable, + ctx: &mut VmCore, + namespace: Arc>, +) -> Result<()> { + let current_instruction = ctx.instructions[ctx.ip - 1]; + + let tail_call = matches!( + current_instruction.op_code, + OpCode::TAILCALL | OpCode::CALLGLOBALTAIL | OpCode::CALLGLOBALTAILNOARITY + ); + + let Executable { + instructions, + spans, + .. + } = program; + let mut bytecode = Vec::new(); + let mut new_spans = Vec::new(); + + // let mut global_offset = 0; + + // Rewrite relative jumps at the top level into absolute jumps. + for (instr, span) in instructions.into_iter().zip(spans) { + let mut depth = 0; + + let offset = bytecode.len(); + + new_spans.extend_from_slice(&span); + bytecode.extend_from_slice(&instr); + + bytecode + .last_mut() + .ok_or_else(throw!(Generic => "Compilation error: empty expression"))? + .op_code = OpCode::POPSINGLE; + + for instruction in &mut bytecode[offset..] { + match instruction { + DenseInstruction { + op_code: OpCode::JMP | OpCode::IF, + payload_size, + } => { + if depth == 0 { + *payload_size = *payload_size + u24::from_usize(offset); + } + } + DenseInstruction { + op_code: OpCode::BIND, + payload_size, + } => { + let mut compiler_guard = ctx.thread.compiler.write(); + if compiler_guard + .symbol_map + .free_list + .recently_freed + .contains(&payload_size.to_usize()) + { + compiler_guard + .symbol_map + .free_list + .recently_freed + .remove(&payload_size.to_usize()); + } + } + DenseInstruction { + op_code: + OpCode::PUSH + | OpCode::CALLGLOBAL + | OpCode::CALLPRIMITIVE + | OpCode::CALLGLOBALTAIL + | OpCode::CALLGLOBALNOARITY + | OpCode::CALLGLOBALTAILNOARITY, + payload_size, + } => { + if depth == 0 { + let compiler_guard = ctx.thread.compiler.read(); + + // TODO: Figure out how to make the recently freed list + // move down in size. Eval is the only way that we could + // reclaim these slots, so assuming there isn't any + // eval, we'll want to make sure this goes down in size. + if compiler_guard + .symbol_map + .free_list + .recently_freed + .contains(&payload_size.to_usize()) + && ctx + .thread + .global_env + .repl_maybe_lookup_idx(payload_size.to_usize()) + == Some(SteelVal::Void) + { + stop!(Generic => "Free identifier: eval (maybe) referenced an identifier before it was bound"); + } + } + } + DenseInstruction { + op_code: OpCode::NEWSCLOSURE | OpCode::PUREFUNC, + .. + } => { + depth += 1; + } + DenseInstruction { + op_code: OpCode::ECLOSURE, + .. + } => { + depth -= 1; + } + _ => {} + } + } + } + + // TODO: Fix the unwrap here + if let Some(last) = bytecode.last_mut() { + last.op_code = OpCode::POPPURE; + } else { + // Push an op code void? + bytecode.push(DenseInstruction { + op_code: OpCode::VOID, + payload_size: crate::core::instructions::u24::from_u32(0), + }); + bytecode.push(DenseInstruction { + op_code: OpCode::POPPURE, + payload_size: crate::core::instructions::u24::from_u32(0), + }); + } + + let function_id = crate::compiler::code_gen::fresh_function_id(); + let function = Gc::new(ByteCodeLambda::new( + function_id as _, + StandardShared::from(bytecode), + 0, + false, + CaptureVec::new(), + )); + ctx.thread + .function_interner + .spans + .insert(function_id as _, Shared::from(new_spans)); + + if tail_call { + ctx.new_handle_tail_call_closure(function, 0).unwrap(); + } else { + ctx.ip -= 1; + ctx.handle_function_call_closure(function, 0).unwrap(); + } + + // Here we're going to install the namespace into the current + // execution environment. We do this by first swapping out + // the existing namespace, and then pushing that onto the stack + // frame under the attachments. When this stack frame is popped + // off, we'll check the marks and reinstall accordingly. + if let Some(last) = ctx.thread.stack_frames.last_mut() { + let mut guard = namespace.lock().unwrap(); + + std::mem::swap(&mut ctx.thread.global_env, &mut guard.env); + std::mem::swap(&mut ctx.thread.compiler, &mut guard.compiler); + + drop(guard); + + match &mut last.attachments { + Some(attachments) => { + attachments.namespace = Some(namespace); + } + None => { + last.attachments = Some(Box::new(StackFrameAttachments { + handler: None, + weak_continuation_mark: None, + namespace: Some(namespace), + })) + } + } + } + + Ok(()) +} + fn eval_program(program: crate::compiler::program::Executable, ctx: &mut VmCore) -> Result<()> { let current_instruction = ctx.instructions[ctx.ip - 1]; @@ -5868,6 +6485,14 @@ fn pop_module_context( Some(Ok(SteelVal::Void)) } +#[steel_derive::context(name = "eval-with-namespace", arity = "Exact(2)")] +fn eval_with_namespace(ctx: &mut VmCore, args: &[SteelVal]) -> Option> { + match eval_with_namespace_impl(ctx, args) { + Ok(_) => None, + Err(e) => Some(Err(e)), + } +} + #[steel_derive::context(name = "eval", arity = "Exact(1)")] fn eval(ctx: &mut crate::steel_vm::vm::VmCore, args: &[SteelVal]) -> Option> { match eval_impl(ctx, args) { diff --git a/crates/steel-core/src/values/closed.rs b/crates/steel-core/src/values/closed.rs index 1d1d16ace..525427ebe 100644 --- a/crates/steel-core/src/values/closed.rs +++ b/crates/steel-core/src/values/closed.rs @@ -2280,8 +2280,8 @@ impl HeapAllocated { } pub struct MarkAndSweepContext<'a> { - queue: &'a mut Vec, - stats: MarkAndSweepStats, + pub(crate) queue: &'a mut Vec, + pub(crate) stats: MarkAndSweepStats, } impl<'a> MarkAndSweepContext<'a> { @@ -2324,7 +2324,7 @@ impl<'a> MarkAndSweepContext<'a> { } #[derive(Debug, Clone, Default)] -struct MarkAndSweepStats { +pub(crate) struct MarkAndSweepStats { object_count: usize, memory_reached_count: usize, vector_reached_count: usize, diff --git a/crates/steel-core/src/values/functions.rs b/crates/steel-core/src/values/functions.rs index e0935f6a3..7521c54ce 100644 --- a/crates/steel-core/src/values/functions.rs +++ b/crates/steel-core/src/values/functions.rs @@ -115,8 +115,6 @@ pub struct ByteCodeLambda { pub(crate) is_multi_arity: bool, - // Store... some amount inline? - // pub(crate) captures: Vec, pub(crate) captures: CaptureVec, // pub(crate) captures: Box<[SteelVal]> @@ -132,8 +130,6 @@ pub struct ByteCodeLambda { #[cfg(feature = "jit2")] pub(crate) super_instructions: Option, - // #[cfg(feature = "jit2")] - // pub(crate) tail_call: bool, } impl PartialEq for ByteCodeLambda { @@ -148,10 +144,7 @@ impl Eq for ByteCodeLambda {} impl core::hash::Hash for ByteCodeLambda { fn hash(&self, state: &mut H) { self.id.hash(state); - // self.body_exp.as_ptr().hash(state); self.arity.hash(state); - - // self.sub_expression_env.as_ptr().hash(state); } } diff --git a/crates/steel-core/src/values/mod.rs b/crates/steel-core/src/values/mod.rs index b7fcbab30..c65430337 100644 --- a/crates/steel-core/src/values/mod.rs +++ b/crates/steel-core/src/values/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod json_vals; pub(crate) mod lazy_stream; pub(crate) mod lists; pub(crate) mod port; +pub(crate) mod reachable; pub(crate) mod recycler; pub(crate) mod structs; pub(crate) mod transducers; diff --git a/crates/steel-core/src/values/reachable.rs b/crates/steel-core/src/values/reachable.rs new file mode 100644 index 000000000..5d3b212ec --- /dev/null +++ b/crates/steel-core/src/values/reachable.rs @@ -0,0 +1,382 @@ +use std::collections::HashSet; + +use crate::values::closed::{HeapRef, MarkAndSweepContext, MarkAndSweepStats}; +use crate::{ + gc::{shared::ShareableMut, GcMut}, + rvals::{OpaqueIterator, SteelComplex, SteelVector}, + steel_vm::vm::{Continuation, ContinuationMark}, + values::lists::List, +}; + +use num_bigint::BigInt; +use num_rational::{BigRational, Rational32}; + +use steel_gen::OpCode; + +use crate::{ + gc::{unsafe_erased_pointers::OpaqueReference, Gc}, + rvals::{ + cycles::BreadthFirstSearchSteelValVisitor, BoxedAsyncFunctionSignature, CustomType, + FunctionSignature, FutureResult, MutFunctionSignature, SteelHashMap, SteelHashSet, + SteelString, Syntax, + }, + steel_vm::vm::BuiltInSignature, + values::functions::ByteCodeLambda, + SteelVal, +}; + +use super::{ + functions::BoxedDynFunction, + lazy_stream::LazyStream, + port::SteelPort, + structs::UserDefinedStruct, + transducers::{Reducer, Transducer}, +}; + +#[derive(Default)] +pub struct GlobalSlotReacher<'a> { + // Use a hashset to check for free slots. + // The idea here is that a collection will traverse + // all active values (excluding roots with this index) + // and we'll check to make sure these are reachable. + // + // If the values are eventually reachable, then we can keep + // iterating until this is completely drained. + // + // If we reach the end of our iteration and this isn't + // drained, whatever is left is now freeable, and we can make + // this as free in the symbol map + slots: HashSet, + + globals: &'a [SteelVal], + + visited: HashSet, + + queue: Vec, +} + +impl<'a> GlobalSlotReacher<'a> { + pub fn find_reachable(globals: &[SteelVal], roots: &mut [SteelVal]) -> HashSet { + let mut recycler = GlobalSlotReacher::default(); + recycler.globals = globals; + + recycler.recycle(roots); + recycler.slots + } + + // TODO: + // Take the global roots, without the shadowed values, and iterate over them, + // push the values back, visit, mark visited, move on. + fn recycle(&mut self, roots: &mut [SteelVal]) { + self.slots.clear(); + + for root in roots { + self.push_back(root.clone()); + } + + self.visit(); + } +} + +impl<'a> BreadthFirstSearchSteelValVisitor for GlobalSlotReacher<'a> { + type Output = (); + + fn default_output(&mut self) -> Self::Output {} + + fn pop_front(&mut self) -> Option { + self.queue.pop() + } + + fn visit(&mut self) -> Self::Output { + use SteelVal::*; + + while let Some(value) = self.pop_front() { + match value { + Closure(c) => self.visit_closure(c), + BoolV(b) => self.visit_bool(b), + NumV(n) => self.visit_float(n), + IntV(i) => self.visit_int(i), + Rational(x) => self.visit_rational(x), + BigRational(x) => self.visit_bigrational(x), + BigNum(b) => self.visit_bignum(b), + Complex(x) => self.visit_complex(x), + CharV(c) => self.visit_char(c), + VectorV(v) => self.visit_immutable_vector(v), + Void => self.visit_void(), + StringV(s) => self.visit_string(s), + FuncV(f) => self.visit_function_pointer(f), + SymbolV(s) => self.visit_symbol(s), + SteelVal::Custom(c) => self.visit_custom_type(c), + HashMapV(h) => self.visit_hash_map(h), + HashSetV(s) => self.visit_hash_set(s), + CustomStruct(c) => self.visit_steel_struct(c), + PortV(p) => self.visit_port(p), + IterV(t) => self.visit_transducer(t), + ReducerV(r) => self.visit_reducer(r), + FutureFunc(f) => self.visit_future_function(f), + FutureV(f) => self.visit_future(f), + StreamV(s) => self.visit_stream(s), + BoxedFunction(b) => self.visit_boxed_function(b), + ContinuationFunction(c) => self.visit_continuation(c), + ListV(l) => self.visit_list(l), + MutFunc(m) => self.visit_mutable_function(m), + BuiltIn(b) => self.visit_builtin_function(b), + MutableVector(b) => self.visit_mutable_vector(b), + BoxedIterator(b) => self.visit_boxed_iterator(b), + SteelVal::SyntaxObject(s) => self.visit_syntax_object(s), + Boxed(b) => self.visit_boxed_value(b), + Reference(r) => self.visit_reference_value(r), + HeapAllocated(b) => self.visit_heap_allocated(b), + Pair(p) => self.visit_pair(p), + ByteVector(b) => self.visit_bytevector(b), + }; + } + } + + fn push_back(&mut self, value: SteelVal) { + // TODO: Determine if all numbers should push back. + match &value { + SteelVal::BoolV(_) + | SteelVal::NumV(_) + | SteelVal::IntV(_) + | SteelVal::CharV(_) + | SteelVal::Void + | SteelVal::StringV(_) + | SteelVal::FuncV(_) + | SteelVal::SymbolV(_) + | SteelVal::FutureFunc(_) + | SteelVal::FutureV(_) + | SteelVal::BoxedFunction(_) + | SteelVal::MutFunc(_) + | SteelVal::BuiltIn(_) + | SteelVal::ByteVector(_) + | SteelVal::BigNum(_) => {} + _ => { + self.queue.push(value); + } + } + } + + fn visit_bytevector(&mut self, _bytevector: crate::rvals::SteelByteVector) -> Self::Output {} + fn visit_bignum(&mut self, _: Gc) -> Self::Output {} + fn visit_complex(&mut self, _: Gc) -> Self::Output {} + fn visit_bool(&mut self, _boolean: bool) -> Self::Output {} + fn visit_boxed_function(&mut self, _function: Gc) -> Self::Output {} + // TODO: Revisit this when the boxed iterator is cleaned up + fn visit_boxed_iterator(&mut self, iterator: GcMut) -> Self::Output { + self.push_back(iterator.read().root.clone()); + } + fn visit_boxed_value(&mut self, boxed_value: GcMut) -> Self::Output { + self.push_back(boxed_value.read().clone()); + } + + fn visit_builtin_function(&mut self, _function: BuiltInSignature) -> Self::Output {} + + fn visit_char(&mut self, _c: char) -> Self::Output {} + fn visit_closure(&mut self, closure: Gc) -> Self::Output { + for capture in closure.captures() { + println!("Visiting capture: {}", capture); + self.push_back(capture.clone()); + } + + if let Some(contract) = closure.get_contract_information().as_ref() { + self.push_back(contract.clone()); + } + + for instruction in closure.body_exp.iter() { + match instruction.op_code { + // If this instruction touches this global variable, + // then we want to mark it as possibly referenced here. + OpCode::CALLGLOBAL + | OpCode::CALLPRIMITIVE + | OpCode::PUSH + | OpCode::CALLGLOBALTAIL + | OpCode::CALLGLOBALNOARITY + | OpCode::CALLGLOBALTAILNOARITY => { + let idx = instruction.payload_size.to_usize(); + + self.slots.insert(idx); + + if self.visited.insert(idx) { + self.push_back(self.globals[idx].clone()); + } + } + _ => {} + } + } + } + fn visit_continuation(&mut self, continuation: Continuation) -> Self::Output { + let continuation = (*continuation.inner.read()).clone(); + + match continuation { + ContinuationMark::Closed(continuation) => { + for value in &continuation.stack { + self.push_back(value.clone()); + } + + for value in &continuation.current_frame.function.captures { + self.push_back(value.clone()); + } + + for frame in &continuation.stack_frames { + for value in &frame.function.captures { + self.push_back(value.clone()); + } + + // if let Some(handler) = &frame.handler { + // self.push_back((*handler.as_ref()).clone()); + // } + + if let Some(handler) = + frame.attachments.as_ref().and_then(|x| x.handler.clone()) + { + self.push_back(handler); + } + } + } + + ContinuationMark::Open(continuation) => { + for value in &continuation.current_stack_values { + self.push_back(value.clone()); + } + + for value in &continuation.current_frame.function.captures { + self.push_back(value.clone()); + } + } + } + } + // TODO: Come back to this + fn visit_custom_type(&mut self, custom_type: GcMut>) -> Self::Output { + let mut queue = MarkAndSweepContext { + queue: &mut self.queue, + stats: MarkAndSweepStats::default(), + }; + + custom_type.read().visit_children(&mut queue); + } + + fn visit_float(&mut self, _float: f64) -> Self::Output {} + + fn visit_function_pointer(&mut self, _ptr: FunctionSignature) -> Self::Output {} + + fn visit_future(&mut self, _future: Gc) -> Self::Output {} + + fn visit_future_function(&mut self, _function: BoxedAsyncFunctionSignature) -> Self::Output {} + + fn visit_hash_map(&mut self, hashmap: SteelHashMap) -> Self::Output { + for (key, value) in hashmap.iter() { + self.push_back(key.clone()); + self.push_back(value.clone()); + } + } + + fn visit_hash_set(&mut self, hashset: SteelHashSet) -> Self::Output { + for value in hashset.iter() { + self.push_back(value.clone()); + } + } + + fn visit_heap_allocated(&mut self, heap_ref: HeapRef) -> Self::Output { + let mut queue = MarkAndSweepContext { + queue: &mut self.queue, + stats: MarkAndSweepStats::default(), + }; + + queue.mark_heap_reference(&heap_ref.strong_ptr()); + } + + fn visit_immutable_vector(&mut self, vector: SteelVector) -> Self::Output { + for value in vector.iter() { + self.push_back(value.clone()); + } + } + fn visit_int(&mut self, _int: isize) -> Self::Output {} + fn visit_rational(&mut self, _: Rational32) -> Self::Output {} + fn visit_bigrational(&mut self, _: Gc) -> Self::Output {} + + fn visit_list(&mut self, list: List) -> Self::Output { + for value in list { + self.push_back(value); + } + } + + fn visit_mutable_function(&mut self, _function: MutFunctionSignature) -> Self::Output {} + + fn visit_mutable_vector(&mut self, vector: HeapRef>) -> Self::Output { + let mut queue = MarkAndSweepContext { + queue: &mut self.queue, + stats: MarkAndSweepStats::default(), + }; + + queue.mark_heap_vector(&vector.strong_ptr()) + } + + fn visit_port(&mut self, _port: SteelPort) -> Self::Output {} + + fn visit_reducer(&mut self, reducer: Gc) -> Self::Output { + match reducer.as_ref().clone() { + Reducer::ForEach(f) => self.push_back(f), + Reducer::Generic(rf) => { + self.push_back(rf.initial_value); + self.push_back(rf.function); + } + _ => {} + } + } + + // TODO: Revisit this + fn visit_reference_value(&mut self, _reference: Gc>) -> Self::Output {} + + fn visit_steel_struct(&mut self, steel_struct: Gc) -> Self::Output { + for field in steel_struct.fields.iter() { + self.push_back(field.clone()); + } + } + + fn visit_stream(&mut self, stream: Gc) -> Self::Output { + self.push_back(stream.initial_value.clone()); + self.push_back(stream.stream_thunk.clone()); + } + + fn visit_string(&mut self, _string: SteelString) -> Self::Output {} + + fn visit_symbol(&mut self, _symbol: SteelString) -> Self::Output {} + + fn visit_syntax_object(&mut self, syntax_object: Gc) -> Self::Output { + if let Some(raw) = syntax_object.raw.clone() { + self.push_back(raw); + } + + self.push_back(syntax_object.syntax.clone()); + } + + fn visit_transducer(&mut self, transducer: Gc) -> Self::Output { + for transducer in transducer.ops.iter() { + match transducer.clone() { + crate::values::transducers::Transducers::Map(m) => self.push_back(m), + crate::values::transducers::Transducers::Filter(v) => self.push_back(v), + crate::values::transducers::Transducers::Take(t) => self.push_back(t), + crate::values::transducers::Transducers::Drop(d) => self.push_back(d), + crate::values::transducers::Transducers::FlatMap(fm) => self.push_back(fm), + crate::values::transducers::Transducers::Flatten => {} + crate::values::transducers::Transducers::Window(w) => self.push_back(w), + crate::values::transducers::Transducers::TakeWhile(tw) => self.push_back(tw), + crate::values::transducers::Transducers::DropWhile(dw) => self.push_back(dw), + crate::values::transducers::Transducers::Extend(e) => self.push_back(e), + crate::values::transducers::Transducers::Cycle => {} + crate::values::transducers::Transducers::Enumerating => {} + crate::values::transducers::Transducers::Zipping(z) => self.push_back(z), + crate::values::transducers::Transducers::Interleaving(i) => self.push_back(i), + crate::values::transducers::Transducers::MapPair(i) => self.push_back(i), + } + } + } + + fn visit_void(&mut self) -> Self::Output {} + + fn visit_pair(&mut self, pair: Gc) -> Self::Output { + self.push_back(pair.car()); + self.push_back(pair.cdr()); + } +}