From 6d2bd3e0a7e597293924b97c96c235bac8e64f14 Mon Sep 17 00:00:00 2001 From: Haibib Kerim Date: Tue, 4 Aug 2026 19:02:38 -0700 Subject: [PATCH 1/3] initial interpreter structure --- blob.fix | 1 + fix/src/interpreter/fixshell.rs | 14 +++++ fix/src/interpreter/grammar.md | 24 ++++++++ fix/src/interpreter/interpreter.rs | 94 ++++++++++++++++++++++++++++++ fix/src/interpreter/mod.rs | 5 ++ fix/src/lib.rs | 2 + fix/src/main.rs | 34 +++++++++-- 7 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 blob.fix create mode 100644 fix/src/interpreter/fixshell.rs create mode 100644 fix/src/interpreter/grammar.md create mode 100644 fix/src/interpreter/interpreter.rs create mode 100644 fix/src/interpreter/mod.rs diff --git a/blob.fix b/blob.fix new file mode 100644 index 0000000..7813681 --- /dev/null +++ b/blob.fix @@ -0,0 +1 @@ +5 \ No newline at end of file diff --git a/fix/src/interpreter/fixshell.rs b/fix/src/interpreter/fixshell.rs new file mode 100644 index 0000000..f5da792 --- /dev/null +++ b/fix/src/interpreter/fixshell.rs @@ -0,0 +1,14 @@ +use kernel::prelude::Box; + +pub trait FixShell: Sized { + type Handle; + + fn create_blob(&self, data: &[u8]) -> Self::Handle; + fn create_tree(&self, data: &[Self::Handle]) -> Self::Handle; + + fn is_blob_obj(handle: Self::Handle) -> bool; + fn is_tree_obj(handle: Self::Handle) -> bool; + + fn get_blob_data(&self, handle: Self::Handle) -> Box<[u8]>; + fn get_tree_data(&self, handle: Self::Handle) -> Box<[Self::Handle]>; +} diff --git a/fix/src/interpreter/grammar.md b/fix/src/interpreter/grammar.md new file mode 100644 index 0000000..8fb3911 --- /dev/null +++ b/fix/src/interpreter/grammar.md @@ -0,0 +1,24 @@ +``` +H ::= O | K | E Handle + | $atom Special/Primitive + +B ::= int literal Blob + | string literal Blob + +T ::= (H*) Tree + +R ::= &B | &T Ref + +O ::= B | T | R Data + +K ::= ^O Thunk (Identify) + | *T Thunk (Apply) + | #T Thunk (Digest) + | ~T Thunk (Select) + | O[B] Thunk (Select 1) + | O[B:B] Thunk (Select N) + +E ::= !K Encode (Strict) + | ?K Encode (Shallow) +``` + diff --git a/fix/src/interpreter/interpreter.rs b/fix/src/interpreter/interpreter.rs new file mode 100644 index 0000000..9053267 --- /dev/null +++ b/fix/src/interpreter/interpreter.rs @@ -0,0 +1,94 @@ +extern crate alloc; +use alloc::collections::BTreeMap; + +use crate::{ + FixShell, Storage, + parser::{Expr, Statement}, +}; +use fixhandle::{Handle, Object}; +use kernel::prelude::*; + +pub struct Interpreter<'a> { + storage: &'a dyn Storage, + context: BTreeMap, +} + +impl<'a> Interpreter<'a> { + pub fn new(storage: &'a dyn Storage) -> Self { + Self { + storage, + context: BTreeMap::new(), + } + } + + pub fn interpret_program(&mut self, program: Vec) { + for statement in program { + match statement { + Statement::Assign { name, expr } => { + let handle = self.interpret(&expr); + self.context.insert(name, handle); + } + Statement::Print(expr) | Statement::Expr(expr) => { + let handle = self.interpret(&expr); + println!("handle: {handle}"); + + if Self::is_blob_obj(handle) { + let contents = self.get_blob_data(handle); + println!("result is a Blob: {contents:?}"); + if contents.len() == 8 { + let bytes: [u8; 8] = (*contents).try_into().unwrap(); + let value = u64::from_le_bytes(bytes); + println!("\tas a u64: {value}"); + } + } + } + } + } + } + + pub fn interpret(&self, expression: &Expr) -> Handle { + match expression { + Expr::String(str) => self.create_blob(str.as_bytes()), + Expr::Number(num) => self.create_blob(&i64::to_le_bytes(*num)), + _ => todo!("support more expressions"), + } + } +} + +impl FixShell for Interpreter<'_> { + type Handle = Handle; + + fn create_blob(&self, data: &[u8]) -> Self::Handle { + self.storage.add_blob(data).into() + } + + fn create_tree(&self, data: &[Self::Handle]) -> Self::Handle { + self.storage.add_tree(data).into() + } + + fn is_blob_obj(handle: Self::Handle) -> bool { + matches!(handle, Handle::Object(Object::Blob(_))) + } + + fn is_tree_obj(handle: Self::Handle) -> bool { + matches!(handle, Handle::Object(Object::Tree(_))) + } + + fn get_blob_data(&self, handle: Self::Handle) -> Box<[u8]> { + let Handle::Object(Object::Blob(blob)) = handle else { + panic!("expected blob handle") + }; + self.storage + .get_blob(blob) + .expect("blob data exists for handle") + } + + fn get_tree_data(&self, handle: Self::Handle) -> Box<[Self::Handle]> { + let Handle::Object(Object::Tree(tree)) = handle else { + panic!("expected tree handle") + }; + self.storage + .get_tree(tree) + .expect("tree data exists for handle") + } +} diff --git a/fix/src/interpreter/mod.rs b/fix/src/interpreter/mod.rs new file mode 100644 index 0000000..7b55799 --- /dev/null +++ b/fix/src/interpreter/mod.rs @@ -0,0 +1,5 @@ +mod fixshell; +mod interpreter; + +pub use fixshell::*; +pub use interpreter::*; diff --git a/fix/src/lib.rs b/fix/src/lib.rs index c09efee..61a3689 100644 --- a/fix/src/lib.rs +++ b/fix/src/lib.rs @@ -1,6 +1,7 @@ #![no_std] pub mod evaluator; +pub mod interpreter; pub mod parser; pub mod runtime; pub mod storage; @@ -11,5 +12,6 @@ pub mod handle { pub use evaluator::*; pub use handle::*; +pub use interpreter::*; pub use runtime::*; pub use storage::*; diff --git a/fix/src/main.rs b/fix/src/main.rs index 88222c1..17214d5 100644 --- a/fix/src/main.rs +++ b/fix/src/main.rs @@ -19,7 +19,7 @@ use alloc::collections::BTreeMap; fn main() { let argv = os::argv(); - // Subcommand dispatch: `fix init` | `fix eval `. + // Subcommand dispatch: `fix init` | `fix eval ` | `fix interpret `. match argv.get(1).map(String::as_str) { Some("init") => init(), Some("eval") => { @@ -31,8 +31,14 @@ fn main() { let filename = argv.get(2).expect("fix eval: expected a command file"); eval_file_parallel(filename); } - Some(other) => panic!("fix: unknown command '{other}' (expected: init | eval )"), - None => panic!("fix: expected a command (init | eval )"), + Some("interpret") => { + let filename = argv.get(2).expect("fix interpret: expected a program file"); + interpret_file(filename); + } + Some(other) => panic!( + "fix: unknown command '{other}' (expected: init | eval | interpret )" + ), + None => panic!("fix: expected a command (init | eval | interpret )"), } kernel::shutdown(); @@ -142,7 +148,6 @@ fn eval_file_parallel(filename: &str) { file.seek(Whence::Start(0)); let mut buf = vec![0; len]; file.read_exact(&mut buf); - let file = core::str::from_utf8(&buf).unwrap(); let lexer = Lexer::new(&file); @@ -223,3 +228,24 @@ fn eval_parallel( Expr::Group(x) => eval_parallel(&evaluator, x, ctx), } } + +fn interpret_file(filename: &str) { + let mut file = File::open(filename, true, false, false, false, false).unwrap(); + let len = file.seek(Whence::End(0)) as usize; + file.seek(Whence::Start(0)); + let mut buf = vec![0; len]; + file.read_exact(&mut buf); + + let file = core::str::from_utf8(&buf).unwrap(); + + let lexer = Lexer::new(&file); + let tokens = lexer.tokenize().unwrap(); + let mut parser = Parser::new(&tokens); + let program = parser.parse_program().unwrap(); + + let runtime = FixOnArca::default(); + let evaluator = Evaluator::new(runtime); + + let mut interpreter = Interpreter::new(evaluator.storage()); + interpreter.interpret_program(program); +} From b59c0469e51c7b7a23517ec5320040f2776605df Mon Sep 17 00:00:00 2001 From: SaibililaA Date: Wed, 5 Aug 2026 18:26:49 -0700 Subject: [PATCH 2/3] Refactor to use interpreter with new grammar. --- addblob.fix | 10 +- blob.fix | 1 - blobref.fix | 4 - fix/src/evaluator.rs | 2 +- fix/src/interpreter/fixshell.rs | 9 +- fix/src/interpreter/grammar.md | 8 +- fix/src/interpreter/interpreter.rs | 91 ++++++----- fix/src/main.rs | 237 +++++------------------------ fix/src/parser/lexer.rs | 58 ++++++- fix/src/parser/parser.rs | 114 ++++++-------- fix/src/parser/token.rs | 27 ++-- 11 files changed, 221 insertions(+), 340 deletions(-) delete mode 100644 blob.fix delete mode 100644 blobref.fix diff --git a/addblob.fix b/addblob.fix index 2105ac4..81913eb 100644 --- a/addblob.fix +++ b/addblob.fix @@ -1,8 +1,2 @@ -x = 2; -y = 3; -add = create_blob("./target/x86_64-unknown-none/addblob"); -add_x_y = create_application_thunk(create_tree(add, x, y)); -z = 1; -sum_x_y = create_strict_encode(add_x_y) -add_xy_z = create_application_thunk(create_tree(add, sum_x_y, z)); -eval(create_strict_encode(add_xy_z)); +(let ((add @"./target/x86_64-unknown-none/addblob")) + !*(add *(add ^&2 3) 1)) \ No newline at end of file diff --git a/blob.fix b/blob.fix deleted file mode 100644 index 7813681..0000000 --- a/blob.fix +++ /dev/null @@ -1 +0,0 @@ -5 \ No newline at end of file diff --git a/blobref.fix b/blobref.fix deleted file mode 100644 index 124f7b3..0000000 --- a/blobref.fix +++ /dev/null @@ -1,4 +0,0 @@ -x = 2; -y = &x; -identify_y = *y; -eval(create_strict_encode(identify_y)); \ No newline at end of file diff --git a/fix/src/evaluator.rs b/fix/src/evaluator.rs index e961f41..1e28e9d 100644 --- a/fix/src/evaluator.rs +++ b/fix/src/evaluator.rs @@ -98,7 +98,7 @@ impl Evaluator { println!("evaluating {handle}"); match handle { Handle::Ref(reference) => self.eval(self.lift(Handle::Ref(reference))), - Handle::Thunk(_) => todo!(), + Handle::Thunk(thunk) => self.think(thunk), Handle::Object(obj) => match obj { Object::Blob(blob) => blob.into(), Object::Tree(tree) => self.eval_tree(tree).into(), diff --git a/fix/src/interpreter/fixshell.rs b/fix/src/interpreter/fixshell.rs index f5da792..520b315 100644 --- a/fix/src/interpreter/fixshell.rs +++ b/fix/src/interpreter/fixshell.rs @@ -5,10 +5,13 @@ pub trait FixShell: Sized { fn create_blob(&self, data: &[u8]) -> Self::Handle; fn create_tree(&self, data: &[Self::Handle]) -> Self::Handle; - - fn is_blob_obj(handle: Self::Handle) -> bool; - fn is_tree_obj(handle: Self::Handle) -> bool; + fn create_ref(handle: Self::Handle) -> Self::Handle; fn get_blob_data(&self, handle: Self::Handle) -> Box<[u8]>; fn get_tree_data(&self, handle: Self::Handle) -> Box<[Self::Handle]>; + + fn create_application_thunk(handle: Self::Handle) -> Self::Handle; + fn create_identification_thunk(handle: Self::Handle) -> Self::Handle; + + fn create_strict_encode(handle: Self::Handle) -> Self::Handle; } diff --git a/fix/src/interpreter/grammar.md b/fix/src/interpreter/grammar.md index 8fb3911..976085d 100644 --- a/fix/src/interpreter/grammar.md +++ b/fix/src/interpreter/grammar.md @@ -1,11 +1,12 @@ ``` -H ::= O | K | E Handle +A ::= O | K | E | L Any | $atom Special/Primitive + | @path Executable path B ::= int literal Blob | string literal Blob -T ::= (H*) Tree +T ::= (A*) Tree R ::= &B | &T Ref @@ -20,5 +21,8 @@ K ::= ^O Thunk (Identify) E ::= !K Encode (Strict) | ?K Encode (Shallow) + +L ::= (let ((name A)) Let + (A)) ``` diff --git a/fix/src/interpreter/interpreter.rs b/fix/src/interpreter/interpreter.rs index 9053267..a39646a 100644 --- a/fix/src/interpreter/interpreter.rs +++ b/fix/src/interpreter/interpreter.rs @@ -1,11 +1,8 @@ extern crate alloc; use alloc::collections::BTreeMap; -use crate::{ - FixShell, Storage, - parser::{Expr, Statement}, -}; -use fixhandle::{Handle, Object}; +use crate::{FixShell, Storage, parser::Expr}; +use fixhandle::*; use kernel::prelude::*; pub struct Interpreter<'a> { @@ -21,36 +18,30 @@ impl<'a> Interpreter<'a> { } } - pub fn interpret_program(&mut self, program: Vec) { - for statement in program { - match statement { - Statement::Assign { name, expr } => { - let handle = self.interpret(&expr); - self.context.insert(name, handle); - } - Statement::Print(expr) | Statement::Expr(expr) => { - let handle = self.interpret(&expr); - println!("handle: {handle}"); - - if Self::is_blob_obj(handle) { - let contents = self.get_blob_data(handle); - println!("result is a Blob: {contents:?}"); - if contents.len() == 8 { - let bytes: [u8; 8] = (*contents).try_into().unwrap(); - let value = u64::from_le_bytes(bytes); - println!("\tas a u64: {value}"); - } - } - } - } - } - } - - pub fn interpret(&self, expression: &Expr) -> Handle { + pub fn interpret(&mut self, expression: &Expr) -> Handle { match expression { Expr::String(str) => self.create_blob(str.as_bytes()), Expr::Number(num) => self.create_blob(&i64::to_le_bytes(*num)), - _ => todo!("support more expressions"), + Expr::Bytes(bytes) => self.create_blob(bytes), + Expr::Identifier(name) => *self.context.get(name).expect("undefined identifier"), + Expr::Ref(object) => Self::create_ref(self.interpret(object)), + Expr::Tree(handles) => { + let handles: Vec = handles.iter().map(|x| self.interpret(x)).collect(); + self.create_tree(&handles) + } + Expr::Application(tree) => Self::create_application_thunk(self.interpret(tree)), + Expr::Identification(tree) => Self::create_identification_thunk(self.interpret(tree)), + Expr::StrictEncode(thunk) => Self::create_strict_encode(self.interpret(thunk)), + Expr::Let { bindings, body } => { + let outer_context = self.context.clone(); + for (name, expr) in bindings { + let handle = self.interpret(expr); + self.context.insert(name.clone(), handle); + } + let handle = self.interpret(body); + self.context = outer_context; + handle + } } } } @@ -66,12 +57,12 @@ impl FixShell for Interpreter<'_> { self.storage.add_tree(data).into() } - fn is_blob_obj(handle: Self::Handle) -> bool { - matches!(handle, Handle::Object(Object::Blob(_))) - } - - fn is_tree_obj(handle: Self::Handle) -> bool { - matches!(handle, Handle::Object(Object::Tree(_))) + fn create_ref(handle: Self::Handle) -> Self::Handle { + match handle { + Handle::Object(Object::Blob(blob)) => Handle::Ref(Ref::Blob(blob)), + Handle::Object(Object::Tree(tree)) => Handle::Ref(Ref::Tree(tree)), + _ => panic!("expected blob or tree handle"), + } } fn get_blob_data(&self, handle: Self::Handle) -> Box<[u8]> { @@ -91,4 +82,28 @@ impl FixShell for Interpreter<'_> { .get_tree(tree) .expect("tree data exists for handle") } + + fn create_application_thunk(handle: Self::Handle) -> Self::Handle { + let Handle::Object(Object::Tree(tree)) = handle else { + panic!("expected tree handle for applicaiton") + }; + Thunk::Application(tree).into() + } + + fn create_identification_thunk(handle: Self::Handle) -> Self::Handle { + Thunk::Identification(match handle { + Handle::Object(Object::Blob(blob)) => Ref::Blob(blob), + Handle::Object(Object::Tree(tree)) => Ref::Tree(tree), + Handle::Ref(reference) => reference, + _ => panic!("expected blob or tree handle"), + }) + .into() + } + + fn create_strict_encode(handle: Self::Handle) -> Self::Handle { + let Handle::Thunk(thunk) = handle else { + panic!("expected thunk for strict encode") + }; + Encode::Strict(thunk).into() + } } diff --git a/fix/src/main.rs b/fix/src/main.rs index 17214d5..097cdfa 100644 --- a/fix/src/main.rs +++ b/fix/src/main.rs @@ -4,7 +4,7 @@ mod parallel_evaluator; mod scheduler; -use kernel::host::fs::{self, File, Whence}; +use kernel::host::fs; use kernel::host::os; use kernel::prelude::*; @@ -12,33 +12,24 @@ use fix::arca::FixOnArca; use fix::parser::*; use fix::*; -extern crate alloc; -use alloc::collections::BTreeMap; - #[kmain] fn main() { let argv = os::argv(); - // Subcommand dispatch: `fix init` | `fix eval ` | `fix interpret `. + // Subcommand dispatch: `fix init` | `fix eval ` `. match argv.get(1).map(String::as_str) { Some("init") => init(), Some("eval") => { - let filename = argv.get(2).expect("fix eval: expected a command file"); - eval_file(filename); + let path = argv.get(2).expect("fix eval: expected a command file"); + eval_file(path); } // test to run the parallel evaluator Some("parallel_eval") => { - let filename = argv.get(2).expect("fix eval: expected a command file"); - eval_file_parallel(filename); - } - Some("interpret") => { - let filename = argv.get(2).expect("fix interpret: expected a program file"); - interpret_file(filename); + let path = argv.get(2).expect("fix eval: expected a command file"); + eval_file_parallel(path); } - Some(other) => panic!( - "fix: unknown command '{other}' (expected: init | eval | interpret )" - ), - None => panic!("fix: expected a command (init | eval | interpret )"), + Some(other) => panic!("fix: unknown command '{other}' (expected: init | eval )"), + None => panic!("fix: expected a command (init | eval "), } kernel::shutdown(); @@ -58,194 +49,50 @@ fn init() { } /// `fix eval `: read, parse, and evaluate a command file. -fn eval_file(filename: &str) { - let mut file = File::open(filename, true, false, false, false, false).unwrap(); - let len = file.seek(Whence::End(0)) as usize; - file.seek(Whence::Start(0)); - let mut buf = vec![0; len]; - file.read_exact(&mut buf); - - let file = core::str::from_utf8(&buf).unwrap(); +fn eval_file(path: &str) { + let file = Lexer::read_file(path).unwrap(); + let file = Lexer::preprocess(core::str::from_utf8(&file).unwrap()).unwrap(); + let tokens = Lexer::new(&file).tokenize().unwrap(); + let program = Parser::new(&tokens).parse_program().unwrap(); - let lexer = Lexer::new(&file); - let tokens = lexer.tokenize().unwrap(); - let mut parser = Parser::new(&tokens); - let program = parser.parse_program().unwrap(); - - let runtime = FixOnArca::default(); - let evaluator = Evaluator::new(runtime); - - let mut context = BTreeMap::new(); - for statement in program { - match statement { - Statement::Assign { name, expr } => { - let result = eval(&evaluator, &expr, &mut context); - context.insert(name, result); - } - Statement::Print(expr) | Statement::Expr(expr) => { - let x = eval(&evaluator, &expr, &mut context); - println!("handle: {x}"); - if let Handle::Object(Object::Blob(blob)) = x { - let contents = evaluator.storage().get_blob(blob).unwrap(); - println!("result is a Blob: {contents:?}"); - if contents.len() == 8 { - let bytes: [u8; 8] = (*contents).try_into().unwrap(); - let value = u64::from_le_bytes(bytes); - println!("\tas a u64: {value}"); - } - } - } - } - } -} - -fn eval(evaluator: &Evaluator, e: &Expr, ctx: &mut BTreeMap) -> Handle { - match e { - Expr::Identifier(x) => *ctx.get(x).expect("undefined identifier"), - Expr::Number(x) => { - let bytes = i64::to_le_bytes(*x); - evaluator.storage().add_blob(&bytes).into() - } - Expr::String(x) => { - let bytes = x.as_bytes(); - evaluator.storage().add_blob(bytes).into() - } - Expr::Call { name, args } => { - let arg_handles: Vec = args.iter().map(|x| eval(evaluator, x, ctx)).collect(); - match name.as_str() { - "create_blob" if let Expr::String(path) = &args.get(0).expect("no path") => { - let mut file = File::open(path, true, false, false, false, false).unwrap(); - let len = file.seek(Whence::End(0)); - file.seek(Whence::Start(0)); - let mut buf = vec![0; len as usize]; - file.read_exact(&mut buf); - core::mem::forget(file); - evaluator.storage().add_blob(&buf).into() - } - "create_tree" => evaluator.storage().add_tree(&arg_handles).into(), - "create_application_thunk" => { - Thunk::Application(arg_handles[0].unwrap_object().unwrap_tree()).into() - } - "create_strict_encode" => Encode::Strict(arg_handles[0].unwrap_thunk()).into(), - "eval" => evaluator.eval(arg_handles[0]), - name => todo!("call {name} {args:?}"), - } - } - Expr::IdentificationThunk(x) => { - Thunk::Identification(eval(evaluator, x, ctx).unwrap_ref()).into() + let evaluator = Evaluator::new(FixOnArca::default()); + let mut interpreter = Interpreter::new(evaluator.storage()); + let result = evaluator.eval(interpreter.interpret(&program)); + + println!("handle: {result}"); + println!("Current handle is: {:?}", result); + if let Handle::Object(Object::Blob(blob)) = result { + let contents = evaluator.storage().get_blob(blob).unwrap(); + println!("result is a Blob: {contents:?}"); + if contents.len() == 8 { + let bytes: [u8; 8] = (*contents).try_into().unwrap(); + println!("\tas a u64: {}", u64::from_le_bytes(bytes)); } - Expr::Ref(reference) => evaluator.lower(eval(evaluator, reference, ctx)), - Expr::Group(x) => eval(evaluator, x, ctx), } } // Jennifer: tons of redundancy but I just didn't want to change original code, // in case errors showed up // the main change is just calling the parallel evaluator and how its passed in -fn eval_file_parallel(filename: &str) { - let mut file = File::open(filename, true, false, false, false, false).unwrap(); - let len = file.seek(Whence::End(0)) as usize; - file.seek(Whence::Start(0)); - let mut buf = vec![0; len]; - file.read_exact(&mut buf); - let file = core::str::from_utf8(&buf).unwrap(); - - let lexer = Lexer::new(&file); - let tokens = lexer.tokenize().unwrap(); - let mut parser = Parser::new(&tokens); - let program = parser.parse_program().unwrap(); +fn eval_file_parallel(path: &str) { + let file = Lexer::read_file(path).unwrap(); + let file = Lexer::preprocess(core::str::from_utf8(&file).unwrap()).unwrap(); + let tokens = Lexer::new(&file).tokenize().unwrap(); + let program = Parser::new(&tokens).parse_program().unwrap(); let runtime = FixOnArca::default(); let evaluator = parallel_evaluator::Evaluator::new(runtime); - - let mut context = BTreeMap::new(); - for statement in program { - match statement { - Statement::Assign { name, expr } => { - let result = eval_parallel(evaluator.as_ref(), &expr, &mut context); - context.insert(name, result); - } - Statement::Print(expr) | Statement::Expr(expr) => { - let x = eval_parallel(evaluator.as_ref(), &expr, &mut context); - println!("handle: {x}"); - if let Handle::Object(Object::Blob(blob)) = x { - let contents = evaluator.storage().get_blob(blob).unwrap(); - println!("result is a Blob: {contents:?}"); - if contents.len() == 8 { - let bytes: [u8; 8] = (*contents).try_into().unwrap(); - let value = u64::from_le_bytes(bytes); - println!("\tas a u64: {value}"); - } - } - } - } - } -} - -fn eval_parallel( - evaluator: ¶llel_evaluator::Evaluator, - e: &Expr, - ctx: &mut BTreeMap, -) -> Handle { - match e { - Expr::Identifier(x) => *ctx.get(x).expect("undefined identifier"), - Expr::Number(x) => { - let bytes = i64::to_le_bytes(*x); - evaluator.storage().add_blob(&bytes).into() - } - Expr::String(x) => { - let bytes = x.as_bytes(); - evaluator.storage().add_blob(bytes).into() - } - Expr::Call { name, args } => { - let arg_handles: Vec = args - .iter() - .map(|x| eval_parallel(evaluator, x, ctx)) - .collect(); - match name.as_str() { - "create_blob" if let Expr::String(path) = &args.get(0).expect("no path") => { - let mut file = File::open(path, true, false, false, false, false).unwrap(); - let len = file.seek(Whence::End(0)); - file.seek(Whence::Start(0)); - let mut buf = vec![0; len as usize]; - file.read_exact(&mut buf); - core::mem::forget(file); - evaluator.storage().add_blob(&buf).into() - } - "create_tree" => evaluator.storage().add_tree(&arg_handles).into(), - "create_application_thunk" => { - Thunk::Application(arg_handles[0].unwrap_object().unwrap_tree()).into() - } - "create_strict_encode" => Encode::Strict(arg_handles[0].unwrap_thunk()).into(), - "eval" => evaluator.eval(arg_handles[0]), - name => todo!("call {name} {args:?}"), - } - } - Expr::IdentificationThunk(x) => { - Thunk::Identification(eval_parallel(&evaluator, x, ctx).unwrap_ref()).into() + let mut interpreter = Interpreter::new(evaluator.storage()); + let result = evaluator.eval(interpreter.interpret(&program)); + + println!("handle: {result}"); + println!("Current handle is: {:?}", result); + if let Handle::Object(Object::Blob(blob)) = result { + let contents = evaluator.storage().get_blob(blob).unwrap(); + println!("result is a Blob: {contents:?}"); + if contents.len() == 8 { + let bytes: [u8; 8] = (*contents).try_into().unwrap(); + println!("\tas a u64: {}", u64::from_le_bytes(bytes)); } - Expr::Ref(reference) => evaluator.lower(eval_parallel(&evaluator, reference, ctx)), - Expr::Group(x) => eval_parallel(&evaluator, x, ctx), } -} - -fn interpret_file(filename: &str) { - let mut file = File::open(filename, true, false, false, false, false).unwrap(); - let len = file.seek(Whence::End(0)) as usize; - file.seek(Whence::Start(0)); - let mut buf = vec![0; len]; - file.read_exact(&mut buf); - - let file = core::str::from_utf8(&buf).unwrap(); - - let lexer = Lexer::new(&file); - let tokens = lexer.tokenize().unwrap(); - let mut parser = Parser::new(&tokens); - let program = parser.parse_program().unwrap(); - - let runtime = FixOnArca::default(); - let evaluator = Evaluator::new(runtime); - - let mut interpreter = Interpreter::new(evaluator.storage()); - interpreter.interpret_program(program); -} +} \ No newline at end of file diff --git a/fix/src/parser/lexer.rs b/fix/src/parser/lexer.rs index 6a7765e..34301d4 100644 --- a/fix/src/parser/lexer.rs +++ b/fix/src/parser/lexer.rs @@ -1,12 +1,16 @@ use super::token::Token; use core::iter::Peekable; use core::str::Chars; +use kernel::host::fs::{File, Whence}; use kernel::prelude::*; pub struct Lexer<'a> { characters: Peekable>, } +// Temporary placeholder until the standard library format is finalized +const STDLIB: &str = "./fix/stdlib/"; + impl<'a> Lexer<'a> { pub fn new(input: &'a str) -> Self { Self { @@ -14,6 +18,50 @@ impl<'a> Lexer<'a> { } } + pub fn preprocess(file: &str) -> Result { + let mut output = String::new(); + let mut characters = file.chars().peekable(); + + while let Some(character) = characters.next() { + match character { + '$' => { + let mut name = String::new(); + while let Some(character) = characters.next_if(|&ch| Self::is_identifier(ch)) { + name.push(character); + } + let program = Self::read_file(&format!("{STDLIB}/{name}"))?; + output.push_str(&format!("0x{}", hex::encode(program))); + } + '@' => { + if characters.next() != Some('"') { + return Err(String::from("expected path after '@'")); + } + let mut path = String::new(); + while let Some(next) = characters.next_if(|&ch| ch != '"') { + path.push(next); + } + if characters.next() != Some('"') { + return Err(String::from("unterminated path")); + } + let program = Self::read_file(&path)?; + output.push_str(&format!("0x{}", hex::encode(program))); + } + character => output.push(character), + } + } + Ok(output) + } + + pub fn read_file(path: &str) -> Result, String> { + let mut file = File::open(path, true, false, false, false, false) + .map_err(|_| format!("could not open {path}"))?; + let len = file.seek(Whence::End(0)) as usize; + file.seek(Whence::Start(0)); + let mut data = vec![0; len]; + file.read_exact(&mut data); + Ok(data) + } + pub fn tokenize(mut self) -> Result, String> { let mut tokens = Vec::new(); loop { @@ -35,13 +83,12 @@ impl<'a> Lexer<'a> { }; let token = match character { - ';' => Token::Semicolon, '(' => Token::LParen, ')' => Token::RParen, - ',' => Token::Comma, - '=' => Token::Equals, '&' => Token::Ampersand, '*' => Token::Asterisk, + '^' => Token::Caret, + '!' => Token::Bang, '"' => { let text = self.take(String::new(), |ch| ch != '"'); if self.characters.next() != Some('"') { @@ -57,6 +104,11 @@ impl<'a> Lexer<'a> { } return Err(String::from("unexpected character: '/'")); } + '0' if self.peek(|character| *character == 'x') => { + self.characters.next(); + let digits = self.take(String::new(), |ch| ch.is_ascii_hexdigit()); + Token::Bytes(hex::decode(&digits).map_err(|error| error.to_string())?) + } // Negative numbers '-' if self.peek(|character| character.is_ascii_digit()) => { let number = self.take(String::new(), |ch| ch.is_ascii_digit()); diff --git a/fix/src/parser/parser.rs b/fix/src/parser/parser.rs index 694b9f0..2034fd6 100644 --- a/fix/src/parser/parser.rs +++ b/fix/src/parser/parser.rs @@ -1,4 +1,4 @@ -use super::{Expr, Statement, Token}; +use super::{Expr, Token}; use kernel::prelude::*; pub struct Parser<'a> { @@ -14,87 +14,59 @@ impl<'a> Parser<'a> { } } - pub fn parse_program(&mut self) -> Result, String> { - let mut program = Vec::new(); - loop { - self.skip_separators(); - if self.peek(self.position) == Some(&Token::Eof) { - break; - } - program.push(self.parse_statement()?); - self.skip_separators(); - } - Ok(program) - } - - pub fn parse_statement(&mut self) -> Result { - // 'print()' is special built-in - match (self.peek(self.position), self.peek(self.position + 1)) { - (Some(Token::Identifier(name)), Some(Token::LParen)) if name == "print" => { - // consume 'print''(' - self.advance(); - self.advance(); - let expr = self.parse_expr()?; - self.expect(&Token::RParen, "expected ')' for print")?; - Ok(Statement::Print(expr)) - } - (Some(Token::Identifier(name)), Some(Token::Equals)) => { - let name = name.clone(); - // consume 'identifier' '=' - self.advance(); - self.advance(); - Ok(Statement::Assign { - name, - expr: self.parse_expr()?, - }) - } - _ => Ok(Statement::Expr(self.parse_expr()?)), - } - } - - fn parse_expr(&mut self) -> Result { - let mut expr = self.parse_primitive()?; - - while self.matches(&Token::LParen) { - let Expr::Identifier(name) = expr else { - return Err(String::from("functions must be named")); - }; - - // arguments for function calls - let mut args = Vec::new(); - if self.peek(self.position) != Some(&Token::RParen) { - loop { - args.push(self.parse_expr()?); - if !self.matches(&Token::Comma) { - break; - } - } - } - self.expect(&Token::RParen, "expected ')' for function call")?; - expr = Expr::Call { name, args }; - } - + pub fn parse_program(&mut self) -> Result { + let expr = self.parse_expr()?; + self.expect(&Token::Eof, "expected end of program")?; Ok(expr) } - fn parse_primitive(&mut self) -> Result { + fn parse_expr(&mut self) -> Result { match self.advance() { - Token::Number(value) => Ok(Expr::Number(value)), + Token::Number(number) => Ok(Expr::Number(number)), + Token::String(string) => Ok(Expr::String(string)), + Token::Bytes(bytes) => Ok(Expr::Bytes(bytes)), Token::Identifier(value) => Ok(Expr::Identifier(value)), - Token::String(value) => Ok(Expr::String(value)), + Token::Ampersand => Ok(Expr::Ref(Box::new(self.parse_expr()?))), + Token::Caret => Ok(Expr::Identification(Box::new(self.parse_expr()?))), + Token::Asterisk => Ok(Expr::Application(Box::new(self.parse_expr()?))), + Token::Bang => Ok(Expr::StrictEncode(Box::new(self.parse_expr()?))), Token::LParen => { - let expr = self.parse_expr()?; - self.expect(&Token::RParen, "expected ')' for grouping")?; - Ok(Expr::Group(Box::new(expr))) + if let Some(Token::Identifier(token)) = self.peek(self.position) + && token == "let" + { + self.advance(); + self.parse_let() + } else { + Ok(Expr::Tree(self.parse_handles()?)) + } } - Token::Ampersand => Ok(Expr::Ref(Box::new(self.parse_expr()?))), - Token::Asterisk => Ok(Expr::IdentificationThunk(Box::new(self.parse_expr()?))), token => Err(format!("unexpected token: {token:?}")), } } - fn skip_separators(&mut self) { - while self.matches(&Token::Semicolon) {} + fn parse_handles(&mut self) -> Result, String> { + let mut handles = Vec::new(); + while !self.matches(&Token::RParen) { + handles.push(self.parse_expr()?); + } + Ok(handles) + } + + fn parse_let(&mut self) -> Result { + self.expect(&Token::LParen, "expected '(' for let bindings")?; + let mut bindings = Vec::new(); + while self.matches(&Token::LParen) { + let Token::Identifier(name) = self.advance() else { + return Err(String::from("expected name in let binding")); + }; + let value = self.parse_expr()?; + self.expect(&Token::RParen, "expected ')' for let binding")?; + bindings.push((name, value)); + } + self.expect(&Token::RParen, "expected ')' for let bindings")?; + let body = Box::new(self.parse_expr()?); + self.expect(&Token::RParen, "expected ')' for let")?; + Ok(Expr::Let { bindings, body }) } fn expect(&mut self, token: &Token, message: &str) -> Result<(), String> { diff --git a/fix/src/parser/token.rs b/fix/src/parser/token.rs index 64e21e4..891f14a 100644 --- a/fix/src/parser/token.rs +++ b/fix/src/parser/token.rs @@ -6,30 +6,29 @@ pub enum Token { Identifier(String), Number(i64), String(String), + Bytes(Vec), LParen, RParen, - Comma, - Semicolon, Ampersand, + Caret, Asterisk, - Equals, + Bang, Eof, } #[derive(Debug, Clone)] pub enum Expr { Number(i64), - Identifier(String), String(String), - Call { name: String, args: Vec }, - Group(Box), + Bytes(Vec), + Identifier(String), Ref(Box), - IdentificationThunk(Box), -} - -#[derive(Debug, Clone)] -pub enum Statement { - Assign { name: String, expr: Expr }, - Print(Expr), - Expr(Expr), + Tree(Vec), + Application(Box), + Identification(Box), + StrictEncode(Box), + Let { + bindings: Vec<(String, Expr)>, + body: Box, + }, } From dcad75af5708074498333010c869853f29c6db32 Mon Sep 17 00:00:00 2001 From: SaibililaA Date: Thu, 6 Aug 2026 17:57:55 -0700 Subject: [PATCH 3/3] Added tests and seperated preprocessing. --- addblob.fix | 2 +- addblob_extended.fix | 84 +++++++--------------- demo.fix | 34 +++++++++ fix/src/evaluator.rs | 2 +- fix/src/interpreter/grammar.md | 3 +- fix/src/main.rs | 124 +++++++++++++++++++++++++++------ fix/src/parser.rs | 2 + fix/src/parser/lexer.rs | 61 +--------------- fix/src/parser/preprocessor.rs | 87 +++++++++++++++++++++++ justfile | 1 + 10 files changed, 257 insertions(+), 143 deletions(-) create mode 100644 demo.fix create mode 100644 fix/src/parser/preprocessor.rs diff --git a/addblob.fix b/addblob.fix index 81913eb..0ce7942 100644 --- a/addblob.fix +++ b/addblob.fix @@ -1,2 +1,2 @@ (let ((add @"./target/x86_64-unknown-none/addblob")) - !*(add *(add ^&2 3) 1)) \ No newline at end of file + !*(add !*(add 2 3) 1)) \ No newline at end of file diff --git a/addblob_extended.fix b/addblob_extended.fix index 7c709a0..f221d6d 100644 --- a/addblob_extended.fix +++ b/addblob_extended.fix @@ -1,57 +1,27 @@ -a = 1; -b = 2; -c = 3; -d = 4; -e = 5; -f = 6; -g = 7; -h = 8; -i = 9; -j = 10; -k = 11; -l = 12; -m = 13; -n = 14; -o = 15; -p = 16; - -add = create_blob("./target/x86_64-unknown-none/addblob"); - -ab = create_application_thunk(create_tree(add, a, b)); -cd = create_application_thunk(create_tree(add, c, d)); -ef = create_application_thunk(create_tree(add, e, f)); -gh = create_application_thunk(create_tree(add, g, h)); -ij = create_application_thunk(create_tree(add, i, j)); -kl = create_application_thunk(create_tree(add, k, l)); -mn = create_application_thunk(create_tree(add, m, n)); -op = create_application_thunk(create_tree(add, o, p)); - -sum_ab = create_strict_encode(ab); -sum_cd = create_strict_encode(cd); -sum_ef = create_strict_encode(ef); -sum_gh = create_strict_encode(gh); -sum_ij = create_strict_encode(ij); -sum_kl = create_strict_encode(kl); -sum_mn = create_strict_encode(mn); -sum_op = create_strict_encode(op); - -lefti = create_application_thunk(create_tree(add, sum_ab, sum_cd)); -righti = create_application_thunk(create_tree(add, sum_ef, sum_gh)); -leftr= create_application_thunk(create_tree(add, sum_ij, sum_kl)); -rightr = create_application_thunk(create_tree(add, sum_mn, sum_op)); - -sum_lefti = create_strict_encode(lefti); -sum_righti = create_strict_encode(righti); -sum_leftr = create_strict_encode(leftr); -sum_rightr = create_strict_encode(rightr); - -left = create_application_thunk(create_tree(add, sum_lefti, sum_leftr)); -right = create_application_thunk(create_tree(add, sum_righti, sum_rightr)); - -sum_left = create_strict_encode(left); -sum_right = create_strict_encode(right); - -final = create_application_thunk(create_tree(add, sum_left, sum_right)); - -eval(create_strict_encode(final)); - +(let ((add @"./target/x86_64-unknown-none/addblob") + (a 1) (b 2) (c 3) (d 4) (e 5) (f 6) (g 7) + (h 8) (i 9) (j 10) (k 11) (l 12) (m 13) + (n 14) (o 15) (p 16)) + !*(add + !*(add + !*(add + !*(add a b) + !*(add c d) + ) + !*(add + !*(add e f) + !*(add g h) + ) + ) + !*(add + !*(add + !*(add i j) + !*(add k l) + ) + !*(add + !*(add m n) + !*(add o p) + ) + ) + ) +) diff --git a/demo.fix b/demo.fix new file mode 100644 index 0000000..2aa9a39 --- /dev/null +++ b/demo.fix @@ -0,0 +1,34 @@ +{- + -- The whole file is interpreted as a single expression. Expressions can be comprised of: + + -- Integer Literals + 42 + + -- String Literals + "hello" + + -- Trees + (1 2 3) + + -- References + &"hello" + + -- Identification Thunks + ^&4 + + -- Application Thunks + *(1 2) + + -- Strict Encodes + !^&2 + + -- Let bindings that tie expressions to variables + (let ((x 42)) x) + + -- @ path executables + @"./target/x86_64-unknown-none/addblob" + + -- Primitives (To be implemented) + $interpret + +-} \ No newline at end of file diff --git a/fix/src/evaluator.rs b/fix/src/evaluator.rs index 1e28e9d..0a62aaa 100644 --- a/fix/src/evaluator.rs +++ b/fix/src/evaluator.rs @@ -98,7 +98,7 @@ impl Evaluator { println!("evaluating {handle}"); match handle { Handle::Ref(reference) => self.eval(self.lift(Handle::Ref(reference))), - Handle::Thunk(thunk) => self.think(thunk), + Handle::Thunk(_) => handle, Handle::Object(obj) => match obj { Object::Blob(blob) => blob.into(), Object::Tree(tree) => self.eval_tree(tree).into(), diff --git a/fix/src/interpreter/grammar.md b/fix/src/interpreter/grammar.md index 976085d..5c9bc33 100644 --- a/fix/src/interpreter/grammar.md +++ b/fix/src/interpreter/grammar.md @@ -22,7 +22,6 @@ K ::= ^O Thunk (Identify) E ::= !K Encode (Strict) | ?K Encode (Shallow) -L ::= (let ((name A)) Let - (A)) +L ::= (let ((name A)*) A) Let ``` diff --git a/fix/src/main.rs b/fix/src/main.rs index 097cdfa..2824c99 100644 --- a/fix/src/main.rs +++ b/fix/src/main.rs @@ -1,9 +1,11 @@ #![no_main] #![no_std] +#![feature(custom_test_frameworks)] +#![test_runner(crate::testing::test_runner)] +#![reexport_test_harness_main = "test_main"] mod parallel_evaluator; mod scheduler; - use kernel::host::fs; use kernel::host::os; use kernel::prelude::*; @@ -12,16 +14,26 @@ use fix::arca::FixOnArca; use fix::parser::*; use fix::*; +#[cfg(test)] +mod testing; + +#[cfg(test)] #[kmain] +fn tests() { + test_main(); +} + +#[cfg_attr(not(test), kmain)] +#[cfg_attr(test, allow(dead_code))] fn main() { let argv = os::argv(); - // Subcommand dispatch: `fix init` | `fix eval ` `. + // Subcommand dispatch: `fix init` | `fix eval `. match argv.get(1).map(String::as_str) { Some("init") => init(), Some("eval") => { let path = argv.get(2).expect("fix eval: expected a command file"); - eval_file(path); + eval_file(path) } // test to run the parallel evaluator Some("parallel_eval") => { @@ -48,16 +60,14 @@ fn init() { println!("initialized empty fix store in .fix"); } -/// `fix eval `: read, parse, and evaluate a command file. -fn eval_file(path: &str) { - let file = Lexer::read_file(path).unwrap(); - let file = Lexer::preprocess(core::str::from_utf8(&file).unwrap()).unwrap(); - let tokens = Lexer::new(&file).tokenize().unwrap(); - let program = Parser::new(&tokens).parse_program().unwrap(); +// Jennifer: tons of redundancy but I just didn't want to change original code, +// in case errors showed up +// the main change is just calling the parallel evaluator and how its passed in +fn eval_file_parallel(path: &str) { + let file = preprocessor::read_file(path).unwrap(); - let evaluator = Evaluator::new(FixOnArca::default()); - let mut interpreter = Interpreter::new(evaluator.storage()); - let result = evaluator.eval(interpreter.interpret(&program)); + let evaluator = parallel_evaluator::Evaluator::new(FixOnArca::default()); + let result = eval_parallel_program(core::str::from_utf8(&file).unwrap(), &evaluator); println!("handle: {result}"); println!("Current handle is: {:?}", result); @@ -71,19 +81,23 @@ fn eval_file(path: &str) { } } -// Jennifer: tons of redundancy but I just didn't want to change original code, -// in case errors showed up -// the main change is just calling the parallel evaluator and how its passed in -fn eval_file_parallel(path: &str) { - let file = Lexer::read_file(path).unwrap(); - let file = Lexer::preprocess(core::str::from_utf8(&file).unwrap()).unwrap(); - let tokens = Lexer::new(&file).tokenize().unwrap(); +fn eval_parallel_program( + source: &str, + evaluator: &Arc>, +) -> Handle { + let processed = Preprocessor::new(source).preprocess().unwrap(); + let tokens = Lexer::new(&processed).tokenize().unwrap(); let program = Parser::new(&tokens).parse_program().unwrap(); - let runtime = FixOnArca::default(); - let evaluator = parallel_evaluator::Evaluator::new(runtime); let mut interpreter = Interpreter::new(evaluator.storage()); - let result = evaluator.eval(interpreter.interpret(&program)); + evaluator.eval(interpreter.interpret(&program)) +} + +// `fix eval `: read command file and print result. +fn eval_file(path: &str) { + let file = preprocessor::read_file(path).unwrap(); + let evaluator = Evaluator::new(FixOnArca::default()); + let result = eval_program(core::str::from_utf8(&file).unwrap(), &evaluator); println!("handle: {result}"); println!("Current handle is: {:?}", result); @@ -95,4 +109,68 @@ fn eval_file_parallel(path: &str) { println!("\tas a u64: {}", u64::from_le_bytes(bytes)); } } -} \ No newline at end of file +} + +// parse, interpret, and evaluate source text. +fn eval_program(source: &str, evaluator: &Evaluator) -> Handle { + let processed = Preprocessor::new(source).preprocess().unwrap(); + let tokens = Lexer::new(&processed).tokenize().unwrap(); + let program = Parser::new(&tokens).parse_program().unwrap(); + + let mut interpreter = Interpreter::new(evaluator.storage()); + evaluator.eval(interpreter.interpret(&program)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn eval_value(source: &str, evaluator: &Evaluator) -> Vec { + match eval_program(source, evaluator) { + Handle::Object(Object::Blob(blob)) => { + evaluator.storage().get_blob(blob).unwrap().into() + } + Handle::Object(Object::Tree(tree)) => tree.len().to_le_bytes().into(), + Handle::Ref(Ref::Blob(blob)) => evaluator.storage().get_blob(blob).unwrap().into(), + Handle::Ref(Ref::Tree(tree)) => tree.len().to_le_bytes().into(), + _ => panic!(), + } + } + + #[test_case] + fn test_number() { + let evaluator = Evaluator::new(FixOnArca::default()); + + { + assert_eq!(eval_value("42", &evaluator), 42i64.to_le_bytes()); + assert_eq!(eval_value("-1", &evaluator), (-1i64).to_le_bytes()); + assert_eq!(eval_value("\"hello\"", &evaluator), b"hello"); + + assert_eq!(eval_value("(1 2 3)", &evaluator), 3i64.to_le_bytes()); + assert_eq!(eval_value("()", &evaluator), 0i64.to_le_bytes()); + + assert_eq!(eval_value("&\"hello\"", &evaluator), b"hello"); + assert_eq!(eval_value("&(1 2 3)", &evaluator), 3i64.to_le_bytes()); + + assert_eq!(eval_value("!^&2", &evaluator), 2i64.to_le_bytes()); + + assert_eq!( + eval_value("(let ((x 42)) x)", &evaluator), + 42i64.to_le_bytes() + ); + assert_eq!( + eval_value("(let ((x 1) (y 2)) (x y))", &evaluator), + 2i64.to_le_bytes() + ); + + assert_eq!( + eval_value("(let ((x 1)) (let ((x 2)) x))", &evaluator), + 2i64.to_le_bytes() + ); + assert_eq!( + eval_value("(let ((x 1)) (let ((y (let ((x 2)) x))) x))", &evaluator), + 1i64.to_le_bytes() + ); + } + } +} diff --git a/fix/src/parser.rs b/fix/src/parser.rs index e2d9f74..91d0abc 100644 --- a/fix/src/parser.rs +++ b/fix/src/parser.rs @@ -1,7 +1,9 @@ pub mod lexer; pub mod parser; +pub mod preprocessor; pub mod token; pub use lexer::Lexer; pub use parser::Parser; +pub use preprocessor::Preprocessor; pub use token::*; diff --git a/fix/src/parser/lexer.rs b/fix/src/parser/lexer.rs index 34301d4..188479a 100644 --- a/fix/src/parser/lexer.rs +++ b/fix/src/parser/lexer.rs @@ -1,16 +1,11 @@ use super::token::Token; -use core::iter::Peekable; -use core::str::Chars; -use kernel::host::fs::{File, Whence}; +use core::{iter::Peekable, str::Chars}; use kernel::prelude::*; pub struct Lexer<'a> { characters: Peekable>, } -// Temporary placeholder until the standard library format is finalized -const STDLIB: &str = "./fix/stdlib/"; - impl<'a> Lexer<'a> { pub fn new(input: &'a str) -> Self { Self { @@ -18,50 +13,6 @@ impl<'a> Lexer<'a> { } } - pub fn preprocess(file: &str) -> Result { - let mut output = String::new(); - let mut characters = file.chars().peekable(); - - while let Some(character) = characters.next() { - match character { - '$' => { - let mut name = String::new(); - while let Some(character) = characters.next_if(|&ch| Self::is_identifier(ch)) { - name.push(character); - } - let program = Self::read_file(&format!("{STDLIB}/{name}"))?; - output.push_str(&format!("0x{}", hex::encode(program))); - } - '@' => { - if characters.next() != Some('"') { - return Err(String::from("expected path after '@'")); - } - let mut path = String::new(); - while let Some(next) = characters.next_if(|&ch| ch != '"') { - path.push(next); - } - if characters.next() != Some('"') { - return Err(String::from("unterminated path")); - } - let program = Self::read_file(&path)?; - output.push_str(&format!("0x{}", hex::encode(program))); - } - character => output.push(character), - } - } - Ok(output) - } - - pub fn read_file(path: &str) -> Result, String> { - let mut file = File::open(path, true, false, false, false, false) - .map_err(|_| format!("could not open {path}"))?; - let len = file.seek(Whence::End(0)) as usize; - file.seek(Whence::Start(0)); - let mut data = vec![0; len]; - file.read_exact(&mut data); - Ok(data) - } - pub fn tokenize(mut self) -> Result, String> { let mut tokens = Vec::new(); loop { @@ -96,14 +47,6 @@ impl<'a> Lexer<'a> { } Token::String(text) } - // Inline comments - '/' => { - if self.characters.next() == Some('/') { - self.take(String::new(), |ch| ch != '\n'); - return self.next_token(); - } - return Err(String::from("unexpected character: '/'")); - } '0' if self.peek(|character| *character == 'x') => { self.characters.next(); let digits = self.take(String::new(), |ch| ch.is_ascii_hexdigit()); @@ -143,7 +86,7 @@ impl<'a> Lexer<'a> { text } - fn is_identifier(character: char) -> bool { + pub fn is_identifier(character: char) -> bool { character.is_ascii_alphabetic() || character == '_' } } diff --git a/fix/src/parser/preprocessor.rs b/fix/src/parser/preprocessor.rs new file mode 100644 index 0000000..a5ca8ae --- /dev/null +++ b/fix/src/parser/preprocessor.rs @@ -0,0 +1,87 @@ +use super::lexer::*; +use core::{iter::Peekable, str::Chars}; +use kernel::host::fs::{File, Whence}; +use kernel::prelude::*; + +// Temporary placeholder until the standard library format is finalized +pub const STDLIB: &str = "./fix/stdlib"; + +pub struct Preprocessor<'a> { + characters: Peekable>, +} + +impl<'a> Preprocessor<'a> { + pub fn new(input: &'a str) -> Self { + Self { + characters: input.chars().peekable(), + } + } + + pub fn preprocess(mut self) -> Result { + let mut output = String::new(); + + while let Some(character) = self.characters.next() { + match character { + // Inline comments + '-' if self.characters.next_if_eq(&'-').is_some() => { + while self.characters.next_if(|&ch| ch != '\n').is_some() {} + self.characters.next(); + } + // Block comments + '{' if self.characters.next_if_eq(&'-').is_some() => { + let mut depth = 1; + while depth > 0 { + match (self.characters.next(), self.characters.peek()) { + (Some('{'), Some(&'-')) => { + depth += 1; + self.characters.next(); + } + (Some('-'), Some(&'}')) => { + depth -= 1; + self.characters.next(); + } + (None, _) => return Err(String::from("unterminated block comment")), + _ => {} + } + } + } + '$' => { + let mut name = String::new(); + while let Some(character) = + self.characters.next_if(|&ch| Lexer::is_identifier(ch)) + { + name.push(character); + } + let program = read_file(&format!("{STDLIB}/{name}"))?; + output.push_str(&format!("0x{}", hex::encode(program))); + } + '@' => { + if self.characters.next() != Some('"') { + return Err(String::from("expected path after '@'")); + } + let mut path = String::new(); + while let Some(next) = self.characters.next_if(|&ch| ch != '"') { + path.push(next); + } + if self.characters.next() != Some('"') { + return Err(String::from("unterminated path")); + } + let program = read_file(&path)?; + output.push_str(&format!("0x{}", hex::encode(program))); + } + character => output.push(character), + } + } + Ok(output) + } +} + +pub fn read_file(path: &str) -> Result, String> { + let mut file = File::open(path, true, false, false, false, false) + .map_err(|_| format!("could not open {path}"))?; + let len = file.seek(Whence::End(0)) as usize; + file.seek(Whence::Start(0)); + let mut data = vec![0; len]; + file.read_exact(&mut data); + Ok(data) +} diff --git a/justfile b/justfile index 6b848df..f3dac7f 100644 --- a/justfile +++ b/justfile @@ -17,6 +17,7 @@ build-all: test: cargo test cargo test -p kernel --target={{target}} + cargo test -p fix --bin fix --target={{target}} build bin: cargo build -p kernel --example={{bin}} --target={{target}} {{release}}