diff --git a/crates/steel-core/src/compiler/compiler.rs b/crates/steel-core/src/compiler/compiler.rs index 72559bc73..aa12dfe37 100644 --- a/crates/steel-core/src/compiler/compiler.rs +++ b/crates/steel-core/src/compiler/compiler.rs @@ -22,7 +22,11 @@ use crate::{ parser::parser::Sources, }; -use std::{borrow::Cow, iter::Iterator}; +use std::{ + borrow::Cow, + iter::Iterator, + sync::{Arc, Mutex}, +}; use std::{ collections::{HashMap, HashSet}, path::PathBuf, @@ -315,7 +319,12 @@ pub struct Compiler { pub(crate) macro_env: FxHashMap, pub(crate) module_manager: ModuleManager, opt_level: OptLevel, - pub(crate) kernel: Option, + + // Note: This is... a bit nasty, and most like we need + // the kernel to actually be cheaply cloneable since + // we need to move the kernel down into the proper environments. + pub(crate) kernel: Option>>, + memoization_table: MemoizationTable, mangled_identifiers: FxHashSet, // Try this out? @@ -337,6 +346,8 @@ pub struct Compiler { // want to have the compiler share everything with the runtime. sources: Sources, builtin_modules: ModuleContainer, + + pub(crate) read_table: steel_parser::lexer::ReadTable, } pub struct SerializableCompiler { @@ -413,6 +424,7 @@ impl Compiler { search_dirs: Vec::new(), sources, builtin_modules, + read_table: std::sync::Arc::new(std::sync::Mutex::new(HashMap::default())), } } @@ -431,7 +443,7 @@ impl Compiler { macro_env, module_manager, opt_level: OptLevel::Three, - kernel: Some(kernel), + kernel: Some(Arc::new(Mutex::new(kernel))), memoization_table: MemoizationTable::new(), mangled_identifiers: FxHashSet::default(), lifted_kernel_environments: HashMap::new(), @@ -441,6 +453,7 @@ impl Compiler { search_dirs: Vec::new(), sources, builtin_modules, + read_table: std::sync::Arc::new(std::sync::Mutex::new(HashMap::default())), } } @@ -516,15 +529,35 @@ impl Compiler { let id = self.sources.add_source(expr_str.clone(), path.clone()); + // let mut read_table = HashMap::new(); + + // Pass in the compiler as well to this function? Do something like that? + // read_table.insert( + // '◊', + // Box::new( + // |lexer: &mut steel_parser::lexer::Lexer, c: char| -> ExprKind { + // println!("Hello world!"); + + // ExprKind::atom("foo") + // }, + // ) as _, + // ); + + let mut read_table = self.read_table.lock().unwrap(); + // Could fail here let parsed: std::result::Result, ParseError> = path .as_ref() .map(|p| Parser::new_from_source(expr_str.as_ref(), p.clone(), Some(id))) .unwrap_or_else(|| Parser::new(expr_str.as_ref(), Some(id))) .without_lowering() + .with_read_table(Some(&mut read_table)) .map(|x| x.and_then(lower_macro_and_require_definitions)) .collect(); + // Otherwise we can't continue on here + drop(read_table); + #[cfg(feature = "profiling")] if log::log_enabled!(target: "pipeline_time", log::Level::Debug) { log::debug!(target: "pipeline_time", "Parsing Time: {:?}", now.elapsed()); @@ -612,7 +645,10 @@ impl Compiler { self.module_manager.add_module( path, &mut self.macro_env, - &mut self.kernel, + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), sources, builtin_modules, ) @@ -630,7 +666,10 @@ impl Compiler { // #[cfg(feature = "modules")] return self.module_manager.compile_main( &mut self.macro_env, - &mut self.kernel, + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), &mut self.sources, exprs, path, @@ -694,13 +733,19 @@ impl Compiler { if let Some(kernel) = self.kernel.as_mut() { // Label anything at the top as well - top level - kernel.load_syntax_transformers(&mut expanded_statements, "top-level".to_string())?; + kernel + .lock() + .unwrap() + .load_syntax_transformers(&mut expanded_statements, "top-level".to_string())?; } for expr in expanded_statements.iter_mut() { expand_kernel_in_env( expr, - self.kernel.as_mut(), + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), self.builtin_modules.clone(), "top-level", )?; @@ -711,7 +756,10 @@ impl Compiler { for expr in expanded_statements.iter_mut() { expand_kernel_in_env( expr, - self.kernel.as_mut(), + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), self.builtin_modules.clone(), "top-level", )?; @@ -738,7 +786,10 @@ impl Compiler { for (module, lifted_env) in &mut self.lifted_kernel_environments { let changed = expand_kernel_in_env_with_change( expr, - self.kernel.as_mut(), + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), self.builtin_modules.clone(), &module, )?; @@ -750,7 +801,10 @@ impl Compiler { expand_kernel_in_env( expr, - self.kernel.as_mut(), + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), self.builtin_modules.clone(), "top-level", )?; @@ -889,13 +943,19 @@ impl Compiler { if let Some(kernel) = self.kernel.as_mut() { // Label anything at the top as well - top level - kernel.load_syntax_transformers(&mut expanded_statements, "top-level".to_string())?; + kernel + .lock() + .unwrap() + .load_syntax_transformers(&mut expanded_statements, "top-level".to_string())?; } for expr in expanded_statements.iter_mut() { expand_kernel_in_env( expr, - self.kernel.as_mut(), + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), self.builtin_modules.clone(), "top-level", )?; @@ -918,7 +978,10 @@ impl Compiler { for (module, lifted_env) in &mut self.lifted_kernel_environments { let changed = expand_kernel_in_env_with_change( expr, - self.kernel.as_mut(), + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), self.builtin_modules.clone(), &module, )?; @@ -930,7 +993,10 @@ impl Compiler { expand_kernel_in_env( expr, - self.kernel.as_mut(), + self.kernel + .as_mut() + .map(|x| x.lock().unwrap()) + .as_deref_mut(), self.builtin_modules.clone(), "top-level", )?; @@ -1189,22 +1255,25 @@ impl Compiler { #[cfg(feature = "profiling")] let opt_time = Instant::now(); - let mut maybe_kernel = None; - if use_kernel { if let Some(kernel) = self.kernel.as_mut() { - kernel.load_program_for_comptime(constants.clone(), &mut expanded_statements)?; + kernel + .lock() + .unwrap() + .load_program_for_comptime(constants.clone(), &mut expanded_statements)?; } } + let mut kernel = self.kernel.as_mut().map(|x| x.lock().unwrap()); + let mut manager = ConstantEvaluatorManager::new( &mut self.memoization_table, constants.clone(), self.opt_level, if use_kernel { - &mut self.kernel + kernel.as_deref_mut() } else { - &mut maybe_kernel + None }, ); @@ -1228,9 +1297,9 @@ impl Compiler { constants, self.opt_level, if use_kernel { - &mut self.kernel + kernel.as_deref_mut() } else { - &mut maybe_kernel + None }, ) .run(expanded_statements)?; diff --git a/crates/steel-core/src/compiler/modules.rs b/crates/steel-core/src/compiler/modules.rs index 7a6b41883..7c2863456 100644 --- a/crates/steel-core/src/compiler/modules.rs +++ b/crates/steel-core/src/compiler/modules.rs @@ -224,7 +224,7 @@ impl ModuleManager { &mut self, path: PathBuf, global_macro_map: &mut FxHashMap, - kernel: &mut Option, + kernel: Option<&mut Kernel>, sources: &mut Sources, builtin_modules: ModuleContainer, ) -> Result<()> { @@ -259,7 +259,7 @@ impl ModuleManager { pub(crate) fn compile_main( &mut self, global_macro_map: &mut FxHashMap, - kernel: &mut Option, + mut kernel: Option<&mut Kernel>, sources: &mut Sources, mut exprs: Vec, path: Option, @@ -287,7 +287,7 @@ impl ModuleManager { &mut self.visited, &mut self.file_metadata, sources, - kernel, + kernel.as_mut().map_or(None, |p| Some(p)), builtin_modules, global_macro_map, &self.custom_builtins, @@ -504,6 +504,8 @@ impl ModuleManager { { let require_for_syntax = require_object.path.get_path(); + // let local_kernel = module_builder.kernel.take(); + let (module, in_scope_macros, mut name_mangler) = Self::find_in_scope_macros( &mut self.compiled_modules, require_for_syntax.as_ref(), @@ -520,7 +522,7 @@ impl ModuleManager { let module_name = module.name.to_str().unwrap().to_string(); - if let Some(kernel) = kernel.as_mut() { + if let Some(kernel) = kernel.as_mut().map_or(None, |p| Some(p)) { if kernel.exported_defmacros(&module_name).is_some() { lifted_kernel_environments.insert( module_name.clone(), @@ -616,7 +618,7 @@ impl ModuleManager { // Expanding the kernel with only these macros... let changed = expand_kernel_in_env_with_change( expr, - kernel.as_mut(), + kernel.as_mut().map_or(None, |p| Some(p)), // We don't need to expand those here ModuleContainer::default(), &module_name, @@ -1746,7 +1748,7 @@ struct ModuleBuilder<'a> { visited: &'a mut FxHashSet, file_metadata: &'a mut crate::HashMap, sources: &'a mut Sources, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, builtin_modules: ModuleContainer, global_macro_map: &'a FxHashMap, custom_builtins: &'a HashMap, @@ -1764,7 +1766,7 @@ impl<'a> ModuleBuilder<'a> { visited: &'a mut FxHashSet, file_metadata: &'a mut crate::HashMap, sources: &'a mut Sources, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, builtin_modules: ModuleContainer, global_macro_map: &'a FxHashMap, custom_builtins: &'a HashMap, @@ -1925,7 +1927,7 @@ impl<'a> ModuleBuilder<'a> { self.visited, self.file_metadata, self.sources, - self.kernel, + self.kernel.as_mut().map_or(None, |p| Some(p)), self.builtin_modules.clone(), self.global_macro_map, self.custom_builtins, @@ -2029,7 +2031,7 @@ impl<'a> ModuleBuilder<'a> { self.visited, self.file_metadata, self.sources, - self.kernel, + self.kernel.as_mut().map_or(None, |p| Some(p)), self.builtin_modules.clone(), self.global_macro_map, self.custom_builtins, @@ -2110,7 +2112,7 @@ impl<'a> ModuleBuilder<'a> { expand_kernel_in_env( expr, - self.kernel.as_mut(), + self.kernel.as_mut().map_or(None, |p| Some(p)), self.builtin_modules.clone(), // Expanding macros in the environment? self.name.to_str().unwrap(), @@ -2146,7 +2148,7 @@ impl<'a> ModuleBuilder<'a> { // expand_kernel(x, self.kernel.as_mut(), self.builtin_modules.clone()) expand_kernel_in_env( expr, - self.kernel.as_mut(), + self.kernel.as_mut().map_or(None, |p| Some(p)), self.builtin_modules.clone(), // Expanding macros in the environment? &self.name.to_str().unwrap(), @@ -2240,7 +2242,7 @@ impl<'a> ModuleBuilder<'a> { // Expanding the kernel with only these macros... let changed = expand_kernel_in_env_with_change( &mut fully_expanded, - self.kernel.as_mut(), + self.kernel.as_mut().map_or(None, |p| Some(p)), // We don't need to expand those here ModuleContainer::default(), &module.name.to_str().unwrap(), @@ -3006,7 +3008,7 @@ impl<'a> ModuleBuilder<'a> { visited: &'a mut FxHashSet, file_metadata: &'a mut crate::HashMap, sources: &'a mut Sources, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, builtin_modules: ModuleContainer, global_macro_map: &'a FxHashMap, custom_builtins: &'a HashMap, @@ -3035,7 +3037,7 @@ impl<'a> ModuleBuilder<'a> { visited: &'a mut FxHashSet, file_metadata: &'a mut crate::HashMap, sources: &'a mut Sources, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, builtin_modules: ModuleContainer, global_macro_map: &'a FxHashMap, custom_builtins: &'a HashMap, @@ -3065,7 +3067,7 @@ impl<'a> ModuleBuilder<'a> { visited: &'a mut FxHashSet, file_metadata: &'a mut crate::HashMap, sources: &'a mut Sources, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, builtin_modules: ModuleContainer, global_macro_map: &'a FxHashMap, custom_builtins: &'a HashMap, diff --git a/crates/steel-core/src/parser/kernel.rs b/crates/steel-core/src/parser/kernel.rs index 5370388d6..fcca895ca 100644 --- a/crates/steel-core/src/parser/kernel.rs +++ b/crates/steel-core/src/parser/kernel.rs @@ -7,18 +7,19 @@ use fxhash::{FxBuildHasher, FxHashMap, FxHashSet}; #[cfg(feature = "sync")] use once_cell::sync::Lazy; -use steel_parser::tokens::TokenType; +use steel_parser::{lexer::Lexer, parser::Parser, tokens::TokenType}; use crate::{ compiler::{ passes::analysis::SemanticAnalysis, program::{BEGIN_FOR_SYNTAX, DEFMACRO}, }, + gc::unsafe_erased_pointers::CustomReference, parser::{ ast::{Atom, Set}, parser::SyntaxObject, }, - rvals::{Result, SteelString}, + rvals::{IntoSteelVal, Result, SteelString}, steel_vm::register_fn::RegisterFn, values::lists::List, }; @@ -95,6 +96,12 @@ impl Default for Kernel { } } +impl<'a> CustomReference for Lexer<'a> {} +crate::custom_reference!(Lexer<'a>); + +impl<'a> CustomReference for Parser<'a> {} +crate::custom_reference!(Parser<'a>); + impl Kernel { pub fn new() -> Self { // Does... sandboxing help here? @@ -138,6 +145,33 @@ impl Kernel { }, ); + // Return unmodified next thing? Convert it + engine.register_fn("raw:read", |parser: &mut Parser| { + parser + .next() + .map(|x| { + x.map_err(|x| x.into()).and_then(|x| { + super::tryfrom_visitor::SyntaxObjectFromExprKind::try_from_expr_kind(x) + }) + }) + .transpose() + }); + + engine.register_fn("raw:eat-next-char!", |parser: &mut Parser| { + parser.lexer().eat() + }); + engine.register_fn("raw:peek-next-char", |parser: &mut Parser| { + parser.lexer().peek() + }); + engine.register_fn("raw:consume-whitespace!", |parser: &mut Parser| { + parser.lexer().consume_whitespace() + }); + + // engine.register_fn("lexer-eat", Lexer::eat); + // engine.register_fn("lexer-consume-whitespace", Lexer::consume_whitespace); + + // Somehow, have the lexer... be the parser? + // Load in parameters. // TODO: Merge this with the path in modules.rs if let Err(err) = @@ -563,6 +597,42 @@ impl Kernel { self.engine.call_function_with_args(function, args.to_vec()) } + pub fn call_reader_macro( + &mut self, + ident: SteelString, + lexer: &mut Lexer, + c: char, + ) -> Result { + // Initialize to a temporary thing just to get this to behave + let mut res: Result = Ok(ExprKind::empty()); + + lexer.temporarily_convert_to_owned(|lexer| { + let mut parser = Parser::new_from_lexer(lexer, false, None); + + res = self + .engine + .with_mut_reference(&mut parser) + .consume(move |engine, mut args| { + args.push(c.into_steelval().unwrap()); + // TODO: Figure out how to map the ident to the right thing. + + // (#%reader-macro-map "env" #\c) -> # + let function = engine.call_function_by_name_with_args( + "#%reader-macro-map", + vec![SteelVal::StringV(ident.clone()), SteelVal::CharV(c)], + )?; + + let result = engine.call_function_with_args(function, args)?; + + TryFromSteelValVisitorForExprKind::root(&result) + }); + + parser.into_lexer() + }); + + res + } + pub(crate) fn expand_syntax_object( &mut self, ident: &InternedString, diff --git a/crates/steel-core/src/parser/parser.rs b/crates/steel-core/src/parser/parser.rs index 07c199991..0b1379971 100644 --- a/crates/steel-core/src/parser/parser.rs +++ b/crates/steel-core/src/parser/parser.rs @@ -228,6 +228,7 @@ impl TryFrom> for SteelVal { Err(SteelErr::new(ErrorKind::UnexpectedToken, "#,@".to_string())) } Dot => Err(SteelErr::new(ErrorKind::UnexpectedToken, ".".to_string())), + ReaderMacroExpression(_) => todo!(), } } } @@ -297,6 +298,7 @@ impl TryFrom for SteelVal { Err(SteelErr::new(ErrorKind::UnexpectedToken, "#,@".to_string()).with_span(span)) } Dot => Err(SteelErr::new(ErrorKind::UnexpectedToken, ".".to_string()).with_span(span)), + ReaderMacroExpression(_) => todo!(), } } } diff --git a/crates/steel-core/src/scheme/kernel.scm b/crates/steel-core/src/scheme/kernel.scm index 006021b59..c02cf0ecd 100644 --- a/crates/steel-core/src/scheme/kernel.scm +++ b/crates/steel-core/src/scheme/kernel.scm @@ -143,7 +143,9 @@ (define options (let ([raw (cdddr unwrapped)]) ; (displayln raw) - (if (empty? raw) raw (map syntax->datum raw)))) + (if (empty? raw) + raw + (map syntax->datum raw)))) (define result (struct-impl struct-name fields options)) (syntax/loc result (syntax-span expr))) @@ -270,10 +272,13 @@ `(set! ,struct-name constructor-proto)) ,(new-make-predicate struct-predicate struct-name fields) - ,@ - (if mutable? (mutable-make-getters struct-name fields) (new-make-getters struct-name fields)) + ,@(if mutable? + (mutable-make-getters struct-name fields) + (new-make-getters struct-name fields)) ;; If this is a mutable struct, generate the setters - ,@(if mutable? (mutable-make-setters struct-name fields) (list)) + ,@(if mutable? + (mutable-make-setters struct-name fields) + (list)) void))))) (define (new-make-predicate struct-predicate-name struct-name fields) @@ -452,7 +457,10 @@ ;; Just register a syntax transformer? (define func (parse-def-syntax unwrapped)) (define name-expr (list-ref unwrapped 1)) - (define name (if (list? name-expr) (list-ref name-expr 0) name-expr)) + (define name + (if (list? name-expr) + (list-ref name-expr 0) + name-expr)) (define originating-file (syntax-originating-file expression)) ;; We'd like to (define env (or originating-file "default")) @@ -482,3 +490,21 @@ (%proto-hash-insert% ,(string->symbol env) (quote ,name) ,func))) (register-macro-transformer! name env))) 'void) + +;; (hash -> hash -> function) +(define *map* (hash)) + +(define (#%reader-macro-map env char) + (~> *map* (hash-get env) (hash-get char))) + +(define (#%add-to-reader-map env char quoted-expr) + (displayln ">> inside the kernel <<") + (define env-map (or (hash-try-get *map* env) (hash))) + (define once-added (hash-insert env-map char (eval quoted-expr))) + + (displayln "Evaluated function:") + (displayln once-added) + + (set! *map* (hash-insert *map* env once-added)) + + (displayln "Finished updating the reader macro map")) diff --git a/crates/steel-core/src/steel_vm/const_evaluation.rs b/crates/steel-core/src/steel_vm/const_evaluation.rs index ca7cdb021..f101e4391 100644 --- a/crates/steel-core/src/steel_vm/const_evaluation.rs +++ b/crates/steel-core/src/steel_vm/const_evaluation.rs @@ -128,7 +128,7 @@ pub struct ConstantEvaluatorManager<'a> { pub(crate) changed: bool, opt_level: OptLevel, _memoization_table: &'a mut MemoizationTable, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, } impl<'a> ConstantEvaluatorManager<'a> { @@ -136,7 +136,7 @@ impl<'a> ConstantEvaluatorManager<'a> { memoization_table: &'a mut MemoizationTable, constant_bindings: HashMap, opt_level: OptLevel, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, ) -> Self { Self { global_env: Rc::new(RefCell::new(ConstantEnv::root(constant_bindings))), @@ -179,7 +179,7 @@ impl<'a> ConstantEvaluatorManager<'a> { &expr_level_set_idents, self.opt_level, self._memoization_table, - self.kernel, + self.kernel.as_mut().map_or(None, |p| Some(p)), ); let mut output = eval.visit(expr)?; self.changed = self.changed || eval.changed; @@ -231,7 +231,7 @@ struct ConstantEvaluator<'a> { changed: bool, opt_level: OptLevel, _memoization_table: &'a mut MemoizationTable, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, scope_contains_define: bool, } @@ -254,7 +254,7 @@ impl<'a> ConstantEvaluator<'a> { expr_level_set_idents: &'a [InternedString], opt_level: OptLevel, memoization_table: &'a mut MemoizationTable, - kernel: &'a mut Option, + kernel: Option<&'a mut Kernel>, ) -> Self { Self { bindings, diff --git a/crates/steel-core/src/steel_vm/engine.rs b/crates/steel-core/src/steel_vm/engine.rs index e1270dc47..3e16d1c2f 100644 --- a/crates/steel-core/src/steel_vm/engine.rs +++ b/crates/steel-core/src/steel_vm/engine.rs @@ -1149,7 +1149,7 @@ impl Engine { pub fn new_sandboxed() -> Self { let mut engine = fresh_kernel_image(true); - engine.virtual_machine.compiler.write().kernel = Some(Kernel::new()); + engine.virtual_machine.compiler.write().kernel = Some(Arc::new(Mutex::new(Kernel::new()))); #[cfg(feature = "profiling")] let now = std::time::Instant::now(); @@ -1382,7 +1382,8 @@ impl Engine { let mut engine = fresh_kernel_image(false); { - engine.virtual_machine.compiler.write().kernel = Some(Kernel::new()); + engine.virtual_machine.compiler.write().kernel = + Some(Arc::new(Mutex::new(Kernel::new()))); } #[cfg(feature = "profiling")] diff --git a/crates/steel-core/src/steel_vm/primitives.rs b/crates/steel-core/src/steel_vm/primitives.rs index d26f6ca84..1c7f014fa 100644 --- a/crates/steel-core/src/steel_vm/primitives.rs +++ b/crates/steel-core/src/steel_vm/primitives.rs @@ -8,6 +8,7 @@ use super::{ CALL_WITH_EXCEPTION_HANDLER_DEFINITION, EVAL_DEFINITION, EVAL_FILE_DEFINITION, EVAL_STRING_DEFINITION, EXPAND_SYNTAX_CASE_DEFINITION, EXPAND_SYNTAX_OBJECTS_DEFINITION, INSPECT_DEFINITION, MACRO_CASE_BINDINGS_DEFINITION, MATCH_SYNTAX_CASE_DEFINITION, + REGISTER_READER_MACRO_DEFINITION, }, }; use crate::{ @@ -1806,6 +1807,7 @@ fn meta_module() -> BuiltInModule { .register_value("raise-error", raise_error_from_error()) .register_native_fn_definition(CALL_CC_DEFINITION) .register_native_fn_definition(EVAL_DEFINITION) + .register_native_fn_definition(REGISTER_READER_MACRO_DEFINITION) .register_native_fn_definition(EVAL_FILE_DEFINITION) .register_native_fn_definition(EXPAND_SYNTAX_OBJECTS_DEFINITION) .register_native_fn_definition(MATCH_SYNTAX_CASE_DEFINITION) diff --git a/crates/steel-core/src/steel_vm/vm.rs b/crates/steel-core/src/steel_vm/vm.rs index 2fad385d4..a6a27c91b 100644 --- a/crates/steel-core/src/steel_vm/vm.rs +++ b/crates/steel-core/src/steel_vm/vm.rs @@ -4632,6 +4632,63 @@ pub fn call_cc(ctx: &mut VmCore, args: &[SteelVal]) -> Option> Some(Ok(SteelVal::ContinuationFunction(continuation))) } +#[steel_derive::context(name = "#%register-reader-macro", arity = "Exact(3)")] +fn register_reader_macro( + ctx: &mut crate::steel_vm::vm::VmCore, + args: &[SteelVal], +) -> Option> { + Some(register_reader_macro_impl(ctx, args)) +} + +// (#%register-reader-macro env char quoted-expr) +fn register_reader_macro_impl(ctx: &mut VmCore, args: &[SteelVal]) -> Result { + // This is gonna suck... + let mut comp = ctx.thread.compiler.write(); + let kernel = comp + .kernel + .as_ref() + .map(|x| Arc::downgrade(&x)) + .ok_or_else( + throw!(Generic => "#%register-reader-macro cannot be used from the kernel environment"), + )?; + + let env: SteelString = SteelString::from_steelval(&args[0])?; + let character: char = char::from_steelval(&args[1])?; + + let mut rt = comp.read_table.lock().unwrap(); + + // The callback... is gonna be nasty + rt.insert( + character, + Box::new(move |lexer, character| { + let kernel = kernel.clone(); + let k = std::sync::Weak::upgrade(&kernel).unwrap(); + + let res = k + .lock() + .unwrap() + .call_reader_macro(env.clone(), lexer, character) + .unwrap(); + + res + }), + ); + + drop(rt); + + comp.kernel.as_mut().map(|x| { + x.lock() + .unwrap() + .engine + .call_function_by_name_with_args("#%add-to-reader-map", args.to_vec()) + .unwrap() + }); + + println!("Finished registering the reader macro"); + + Ok(SteelVal::Void) +} + fn eval_impl(ctx: &mut crate::steel_vm::vm::VmCore, args: &[SteelVal]) -> Result { let expr = crate::parser::ast::TryFromSteelValVisitorForExprKind::root(&args[0])?; // TODO: Looks like this isn't correctly parsing / pushing down macros! @@ -4951,6 +5008,7 @@ pub(crate) fn expand_syntax_case_impl(_ctx: &mut VmCore, args: &[SteelVal]) -> R }; if bindings.is_empty() && binding_kind.is_empty() { + // This actually still needs to be converted to syntax. return Ok(args[0].clone()); } diff --git a/crates/steel-parser/src/lexer.rs b/crates/steel-parser/src/lexer.rs index e461685ee..4ece1497b 100644 --- a/crates/steel-parser/src/lexer.rs +++ b/crates/steel-parser/src/lexer.rs @@ -37,6 +37,7 @@ pub struct Lexer<'a> { queued: Option>>, token_start: usize, token_end: usize, + read_table: Option>, } impl<'a> Lexer<'a> { @@ -47,10 +48,34 @@ impl<'a> Lexer<'a> { queued: None, token_start: 0, token_end: 0, + read_table: None, } } - fn eat(&mut self) -> Option { + // Convert this lexer into an owned lexer, and any resulting mutations + // will get copied _back_ onto the source lexer. + pub fn temporarily_convert_to_owned(&mut self, thunk: impl FnOnce(Lexer<'a>) -> Lexer<'a>) { + let mut staged = Lexer { + source: "", + chars: "".chars().peekable(), + queued: None, + token_start: 0, + token_end: 0, + read_table: None, + }; + + // Swap the staged with the owned one, which should give us the ability + // to build a real parser on top of the lexer. + std::mem::swap(&mut staged, self); + let res = thunk(staged); + *self = res; + } + + pub fn peek(&mut self) -> Option { + self.chars.peek().copied() + } + + pub fn eat(&mut self) -> Option { if let Some(c) = self.chars.next() { self.token_end += c.len_utf8(); Some(c) @@ -60,7 +85,7 @@ impl<'a> Lexer<'a> { } // Consume characters until the next non whitespace input - fn consume_whitespace(&mut self) { + pub fn consume_whitespace(&mut self) { while let Some(&c) = self.chars.peek() { if c.is_whitespace() { self.eat(); @@ -560,6 +585,20 @@ impl<'a> Lexer<'a> { } } +pub type ReadTable = std::sync::Arc< + std::sync::Mutex< + std::collections::HashMap< + char, + Box, char) -> crate::ast::ExprKind + Send + Sync>, + >, + >, +>; + +pub type ReadTableRef<'a> = &'a mut std::collections::HashMap< + char, + Box, char) -> crate::ast::ExprKind + Send + Sync>, +>; + pub struct TokenStream<'a> { pub(crate) lexer: Lexer<'a>, skip_comments: bool, @@ -567,6 +606,18 @@ pub struct TokenStream<'a> { } impl<'a> TokenStream<'a> { + pub fn new_from_lexer( + lexer: Lexer<'a>, + skip_comments: bool, + source_id: Option, + ) -> Self { + Self { + lexer, + skip_comments, + source_id, + } + } + pub fn new(input: &'a str, skip_comments: bool, source_id: Option) -> Self { let (_, char_offset, bytes_offset) = strip_shebang_line(input); @@ -586,6 +637,11 @@ impl<'a> TokenStream<'a> { res } + pub fn with_read_table(mut self, read_table: Option>) -> Self { + self.lexer.read_table = read_table; + self + } + pub fn into_owned>(self, adapter: F) -> OwnedTokenStream<'a, T, F> { OwnedTokenStream { stream: self, @@ -656,7 +712,9 @@ pub enum TokenError { } impl<'a> Iterator for Lexer<'a> { + // Lets... see if we can return an ExprKind... type Item = Result>>; + // type Item = Result>, crate::ast::ExprKind>>; fn next(&mut self) -> Option { if let Some(t) = self.queued.take() { @@ -667,7 +725,30 @@ impl<'a> Iterator for Lexer<'a> { self.token_start = self.token_end; - match self.chars.peek() { + let char = self.chars.peek().copied(); + + match char { + Some(char) + if self + .read_table + .as_mut() + .map(|x| x.contains_key(&char)) + .unwrap_or_default() => + { + self.eat(); + let mut let_me_do_it = std::mem::take(&mut self.read_table); + let callback = let_me_do_it.as_mut().unwrap().get_mut(&char).unwrap(); + + // Yoink out the callback for the duration of the call + let res = (callback)(self, char); + + self.read_table = let_me_do_it; + + // Horrendous hack, but perhaps its a price we have to pay for now. + // We don't want this to be recursive in general. + Some(Ok(TokenType::ReaderMacroExpression(Box::new(res)))) + } + Some(';') => { self.eat(); self.read_rest_of_line(); @@ -676,7 +757,7 @@ impl<'a> Iterator for Lexer<'a> { Some('"') => Some(self.read_string()), - Some(&paren @ ('(' | '[' | '{')) => { + Some(paren @ ('(' | '[' | '{')) => { self.eat(); let kind = match paren { '[' => Paren::Square, @@ -686,7 +767,7 @@ impl<'a> Iterator for Lexer<'a> { Some(Ok(TokenType::OpenParen(kind, None))) } - Some(&paren @ (')' | ']' | '}')) => { + Some(paren @ (')' | ']' | '}')) => { self.eat(); let kind = match paren { ']' => Paren::Square, @@ -746,7 +827,7 @@ impl<'a> Iterator for Lexer<'a> { Some(token) } - Some(c) if !c.is_whitespace() && !c.is_ascii_digit() || *c == '_' => { + Some(c) if !c.is_whitespace() && !c.is_ascii_digit() || c == '_' => { Some(self.read_word()) } Some(c) if c.is_ascii_digit() => Some(self.read_number()), diff --git a/crates/steel-parser/src/parser.rs b/crates/steel-parser/src/parser.rs index 3d28fea89..e5639a3f7 100644 --- a/crates/steel-parser/src/parser.rs +++ b/crates/steel-parser/src/parser.rs @@ -17,7 +17,7 @@ use crate::{ RAW_UNSYNTAX_SPLICING, REQUIRE, RETURN, SET, SYNTAX_QUOTE, UNQUOTE, UNQUOTE_SPLICING, }, interner::InternedString, - lexer::{OwnedTokenStream, ToOwnedString, TokenStream}, + lexer::{OwnedTokenStream, ReadTableRef, ToOwnedString, TokenStream}, span::Span, tokens::{Paren, ParenMod, Token, TokenType}, }; @@ -314,6 +314,15 @@ impl<'a> Parser<'a> { Parser::new(expr, SourceId::none()).collect() } + pub fn lexer(&mut self) -> &mut crate::lexer::Lexer<'a> { + &mut self.tokenizer.stream.lexer + } + + pub fn with_read_table(mut self, read_table: Option>) -> Self { + self.tokenizer.stream = self.tokenizer.stream.with_read_table(read_table); + self + } + pub fn parse_without_lowering(expr: &str) -> Result> { Parser::new(expr, SourceId::none()) .without_lowering() @@ -355,6 +364,30 @@ impl<'a> Parser<'a> { } } + pub fn into_lexer(self) -> crate::lexer::Lexer<'a> { + self.tokenizer.stream.lexer + } + + pub fn new_from_lexer( + lexer: crate::lexer::Lexer<'a>, + skip_comments: bool, + source_id: Option, + ) -> Self { + Parser { + tokenizer: TokenStream::new_from_lexer(lexer, skip_comments, source_id) + .into_owned(InternString), + quote_stack: Vec::new(), + quasiquote_depth: 0, + quote_context: false, + shorthand_quote_stack: Vec::new(), + source_name: None, + context: Vec::new(), + comment_buffer: Vec::new(), + collecting_comments: false, + keep_lists: false, + } + } + pub fn without_lowering(mut self) -> Self { self.keep_lists = true; self @@ -1019,6 +1052,10 @@ impl<'a> Parser<'a> { } } + TokenType::ReaderMacroExpression(e) => { + current_frame.push(*e)?; + } + _ => { if let TokenType::Quote = &token.ty { // self.quote_stack.push(current_frame.exprs.len()); @@ -1144,6 +1181,10 @@ impl<'a> Parser<'a> { continue; } + TokenType::ReaderMacroExpression(e) => { + maybe_return![Ok(*e)]; + } + TokenType::DatumComment => { datum_comments.push(res.span); } diff --git a/crates/steel-parser/src/tokens.rs b/crates/steel-parser/src/tokens.rs index e5c7af967..469963bf7 100644 --- a/crates/steel-parser/src/tokens.rs +++ b/crates/steel-parser/src/tokens.rs @@ -58,8 +58,6 @@ impl Paren { } } -// TODO the character parsing is not quite right -// need to make sure that we can handle cases like "#\SPACE" or "#\a" but not "#\applesauce" #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum TokenType { OpenParen(Paren, Option), @@ -92,11 +90,12 @@ pub enum TokenType { Identifier(S), Keyword(S), Number(Box), - // TODO: Consider using Arc here instead to save - // on copies StringLiteral(Arc), Dot, Error, + + // Hack in... crappy returning of values + ReaderMacroExpression(Box), } impl TokenType { @@ -344,6 +343,7 @@ impl<'a> TokenType> { QuoteSyntax => QuoteSyntax, UnquoteSpliceSyntax => UnquoteSpliceSyntax, Dot => Dot, + ReaderMacroExpression(e) => ReaderMacroExpression(e), } } @@ -382,6 +382,7 @@ impl<'a> TokenType> { QuoteSyntax => QuoteSyntax, UnquoteSpliceSyntax => UnquoteSpliceSyntax, Dot => Dot, + ReaderMacroExpression(e) => ReaderMacroExpression(e), } } } @@ -448,6 +449,7 @@ impl fmt::Display for TokenType { Set => write!(f, "set!"), Require => write!(f, "require"), Dot => write!(f, "."), + ReaderMacroExpression(e) => write!(f, "{}", e), } } }