diff options
Diffstat (limited to 'src/lib.rs')
| -rw-r--r-- | src/lib.rs | 339 |
1 files changed, 14 insertions, 325 deletions
@@ -3,330 +3,19 @@ mod syntactic_token; mod semantic_token; mod tokenizer; mod error; - -pub use addressing::*; -pub use syntactic_token::*; -pub use semantic_token::*; -pub use error::*; -pub use tokenizer::*; - -use std::collections::hash_map::{HashMap, Entry}; -use std::mem::take; - -// On Unicode support: Work with characters, not bytes. This will eventually be -// used in Verdant and Doctrine, and it'd be nice to be able to support other languages. -// The only reason to work with bytes over characters would be for a minor decrease in complexity. -// Only support the assembly of files of up to 64kB. If assets need to be tacked on the end, -// it can be done by another program. The VM will only be able to access the first 64kB of a file anyway. -// Treat \t as a space, have it be a single character. - -// First, turn the program source code into a vector of SyntacticTokens. These -// each contain a SourceLocation, and the type and value of the token. Every single -// non-whitespace character of the program needs to be wrapped in a SyntacticToken. -// The program source code can be accurately reconstructed from this list of -// SyntacticTokens, and when I write GRID, if the mouse is hovering over any point -// in the program listing, I'll be able to determine the exact token that is being hovered. -// For macros, hovering over any character belonging to a macro definition will -// highlight the entire macro definition, and also the currently-hovered body token -// if there is one. Clicking the body token will bring up more information. - -// The SyntacticTokens will be collected into a vector, with label and macro definition -// being constructed as we go. Label definitions are easy, I only need to note down the -// names of the labels in order to validate label references in a later step. If a label -// name has already been defined, tag the token with an error. If a macro name has already -// been defined, tag the token with an error. -// Collect children into macro definitions. This makes sense. - -// Step 2 is to generate bytecode, converting SyntacticTokens into SemanticTokens. -// Label and macro definitions need to contain a list of usizes to references. -// Macro definitions need to contain the body tokens as SemanticTokens. -// Label and macro references need to point to their parents. -// Can I stream-convert tokens from Syntactic to Semantic? -// Each SynToken gets converted to a SemToken? Yeah. - -// I want to change the parser to be a multi-stage struct thing, holding its own state. - -enum SymbolDefinition { Macro(usize), Label(usize) } - -pub fn parse(source_code: &str) { - use SyntacticTokenType as Syn; - use SemanticTokenType as Sem; - - // ============================ STEP 1 ============================ - // Convert the source code into a sorted vector of syntactic tokens and a - // map of symbol definitions. - // ================================================================ - println!("[DEBUG] STEP 1: Parse source code into syntactic tokens"); - let mut syntactic_tokens: Vec<SyntacticToken> = Vec::new(); - let mut symbol_definitions: HashMap<String,SymbolDefinition> = HashMap::new(); - let mut macro_bodies: HashMap<usize, Vec<SyntacticToken>> = HashMap::new(); - let mut macro_definition: Option<usize> = None; - let mut macro_definition_body_tokens: Vec<SyntacticToken> = Vec::new(); - - for mut token in TokenIterator::from_str(source_code) { - if let Some(mdt) = macro_definition { - token.use_in_macro_body(); - let terminate = token.is_macro_terminator(); - macro_definition_body_tokens.push(token); - if terminate { - macro_bodies.insert(mdt, take(&mut macro_definition_body_tokens)); - macro_definition = None; - } - } else { - if let Syn::MacroDefinition(ref name) = token.r#type { - macro_definition = Some(syntactic_tokens.len()); - match symbol_definitions.entry(name.to_string()) { - Entry::Occupied(_) => {token.set_error(Error::DuplicateDefinition);} - Entry::Vacant(v) => {v.insert(SymbolDefinition::Macro(syntactic_tokens.len()));} - } - } else if let Syn::LabelDefinition(ref name) = token.r#type { - match symbol_definitions.entry(name.to_string()) { - Entry::Occupied(_) => {token.set_error(Error::DuplicateDefinition);} - Entry::Vacant(v) => {v.insert(SymbolDefinition::Label(syntactic_tokens.len()));} - } - } else if token.is_macro_terminator() { - token.set_error(Error::OrphanedMacroTerminator); - } - syntactic_tokens.push(token); - } - } - - - // ============================ STEP 2 ============================ - // Convert syntactic tokens into semantic tokens, resolving label and macro - // references in the process. - // ================================================================ - println!("[DEBUG] STEP 2: Resolve label and macro references"); - let syntactic_token_count = syntactic_tokens.len(); - let mut semantic_tokens = Vec::new(); - let mut semantic_macro_bodies: HashMap<usize, Vec<SemanticToken>> = HashMap::new(); - - for (i, mut syn_token) in syntactic_tokens.into_iter().enumerate() { - let sem_token_type = if let Some(err) = syn_token.error { - // Translate over any existing syntax errors - Sem::Error(syn_token.r#type, err) - } else { - match syn_token.r#type { - Syn::Reference(ref name) => { - match symbol_definitions.get(name) { - Some(SymbolDefinition::Macro(addr)) => Sem::MacroReference(*addr), - Some(SymbolDefinition::Label(addr)) => Sem::LabelReference(*addr), - None => Sem::Error(syn_token.r#type, Error::UnresolvedReference), - } - } - Syn::LabelDefinition(name) => { - let label_definition = LabelDefinition { - name, address: 0, references: Vec::new() }; - Sem::LabelDefinition(label_definition) - } - Syn::MacroDefinition(name) => { - let mut sem_body_tokens = Vec::new(); - // Iterate over every token in the body of the macro definition, - // converting each one to a semantic token. - for syn_body_token in macro_bodies.remove(&i).unwrap() { - let sem_body_token_type = if let Some(err) = syn_body_token.error { - // Translate over any existing syntax errors - Sem::Error(syn_body_token.r#type, err) - } else { - match syn_body_token.r#type { - Syn::Reference(ref name) => match symbol_definitions.get(name) { - Some(SymbolDefinition::Macro(addr)) => Sem::MacroReference(*addr), - Some(SymbolDefinition::Label(addr)) => Sem::LabelReference(*addr), - None => Sem::Error(syn_body_token.r#type, Error::UnresolvedReference), - }, - - Syn::LabelDefinition(_) => unreachable!(), - Syn::MacroDefinition(_) => unreachable!(), - Syn::MacroTerminator => { - syn_token.source_location.end = - syn_body_token.source_location.end; - Sem::MacroTerminator - }, - - Syn::Pad(v) => Sem::Pad(v), - Syn::Byte(v) => Sem::Byte(v), - Syn::Short(v) => Sem::Short(v), - Syn::Instruction(v) => Sem::Instruction(v), - - Syn::Comment => Sem::Comment, - } - }; - let sem_body_token = SemanticToken { - r#type: sem_body_token_type, - source_location: syn_body_token.source_location, - bytecode_location: BytecodeLocation::zero(), - }; - sem_body_tokens.push(sem_body_token); - } - semantic_macro_bodies.insert(i, sem_body_tokens); - let macro_definition = MacroDefinition { - name, body_tokens: Vec::new(), references: Vec::new() }; - Sem::MacroDefinition(macro_definition) - } - Syn::MacroTerminator => unreachable!(), - - Syn::Pad(v) => Sem::Pad(v), - Syn::Byte(v) => Sem::Byte(v), - Syn::Short(v) => Sem::Short(v), - Syn::Instruction(v) => Sem::Instruction(v), - - Syn::Comment => Sem::Comment, - } - }; - let sem_token = SemanticToken { - r#type: sem_token_type, - source_location: syn_token.source_location, - bytecode_location: BytecodeLocation::zero(), - }; - semantic_tokens.push(sem_token); - } - assert_eq!(syntactic_token_count, semantic_tokens.len()); - - - // ============================ STEP 3 ============================ - // Iterate over each semantic token, generating bytecode. - // ================================================================ - println!("[DEBUG] STEP 3: Generate bytecode"); - let mut bytecode: Vec<u8> = Vec::new(); - // Map each label token to a list of bytecode addresses to populate - let mut label_reference_addresses: HashMap<usize, Vec<u16>> = HashMap::new(); - // Map each label or macro definition token to a list of reference token pointers - let mut references: HashMap<usize, Vec<usize>> = HashMap::new(); - - macro_rules! addr {() => {bytecode.len() as u16};} - macro_rules! push_u8 {($v:expr) => {bytecode.push($v); 1};} - macro_rules! push_u16 {($v:expr) => {bytecode.extend_from_slice(&$v.to_be_bytes()); 2};} - macro_rules! pad {($p:expr) => {bytecode.resize(bytecode.len() + $p as usize, 0); $p as u16};} - - for (i, sem_token) in semantic_tokens.iter_mut().enumerate() { - let start_addr = addr!(); - let byte_length: u16 = match &mut sem_token.r#type { - Sem::LabelReference(addr) => { - references.entry(*addr).or_default().push(i); - label_reference_addresses.entry(*addr).or_default().push(addr!()); - push_u16!(0u16); 2 - }, - Sem::MacroReference(addr) => { - references.entry(*addr).or_default().push(i); - let mut macro_byte_length: u16 = 0; - for body_token in semantic_macro_bodies.get(addr).unwrap() { - macro_byte_length += match &body_token.r#type { - Sem::LabelReference(addr) => { - label_reference_addresses.entry(*addr).or_default().push(addr!()); - push_u16!(0u16); 2 - }, - Sem::MacroReference(_) => todo!(), - - Sem::LabelDefinition(_) => unreachable!(), - Sem::MacroDefinition(_) => unreachable!(), - - Sem::Pad(p) => { pad!(*p); *p }, - Sem::Byte(b) => { push_u8!(*b); 1 }, - Sem::Short(s) => { push_u16!(*s); 2 }, - Sem::Instruction(b) => { push_u8!(*b); 1 }, - - Sem::MacroTerminator => 0, - Sem::Comment => 0, - Sem::Error(..) => 0, - }; - } - macro_byte_length - }, - - Sem::LabelDefinition(definition) => {definition.address=addr!(); 1}, - Sem::MacroDefinition(_) => 0, - - Sem::Pad(p) => { pad!(*p); *p }, - Sem::Byte(b) => { push_u8!(*b); 1 }, - Sem::Short(s) => { push_u16!(*s); 2 }, - Sem::Instruction(b) => { push_u8!(*b); 1 }, - - Sem::MacroTerminator => unreachable!(), - Sem::Comment => 0, - Sem::Error(..) => 0, - }; - sem_token.bytecode_location.start = start_addr; - sem_token.bytecode_location.length = byte_length; - } - - - // ============================ STEP 4 ============================ - // Fill in addresses for label references. - // ================================================================ - println!("[DEBUG] STEP 4: Fill in values for label references"); - for (label_i, slots) in label_reference_addresses.iter() { - if let Sem::LabelDefinition(LabelDefinition { address, .. }) = semantic_tokens[*label_i].r#type { - let [h,l] = address.to_be_bytes(); - for slot in slots { - bytecode[*slot as usize] = h; - bytecode[slot.wrapping_add(1) as usize] = l; - } - } else { - unreachable!() - } - } - - // ============================ STEP 5 ============================ - // Move references and macro body tokens into label and macro definitions. - // ================================================================ - println!("[DEBUG] STEP 5: Move information into label and macro definition tokens"); - for (i, token) in semantic_tokens.iter_mut().enumerate() { - if let Sem::MacroDefinition(macro_definition) = &mut token.r#type { - macro_definition.body_tokens = semantic_macro_bodies.remove(&i).unwrap(); - if let Some(macro_references) = references.remove(&i) { - macro_definition.references = macro_references; - } - } else if let Sem::LabelDefinition(label_definition) = &mut token.r#type { - if let Some(label_references) = references.remove(&i) { - label_definition.references = label_references; - } - } - } - assert_eq!(references.len(), 0); - - - // ============================ STEP 6 ============================ - // Remove trailing null-bytes from the bytecode. - // ================================================================ - println!("[DEBUG] STEP 6: Trim trailing null bytes"); - if let Some(final_nonnull_byte) = bytecode.iter().rposition(|b| *b != 0) { - let truncated_length = final_nonnull_byte + 1; - let removed_byte_count = bytecode.len() - truncated_length; - if removed_byte_count > 0 { - println!("[INFO] Removed {removed_byte_count} trailing null bytes from assembled bytecode"); - bytecode.truncate(truncated_length); - } - } - - - for token in &semantic_tokens { - if let Sem::MacroDefinition(macro_definition) = &token.r#type { - for body_token in ¯o_definition.body_tokens { - if let Sem::Error(_, err) = body_token.r#type { - println!("[ERROR] (in macro '{}') {err:?} at {}:{}..{}:{}", - macro_definition.name, - body_token.source_location.start.line, - body_token.source_location.start.column, - body_token.source_location.end.line, - body_token.source_location.end.column, - ) - } - } - } else if let Sem::Error(_, err) = token.r#type { - println!("[ERROR {}:{}-{}:{}] {err:?}", - token.source_location.start.line, - token.source_location.start.column, - token.source_location.end.line, - token.source_location.end.column, - ) - } - } - - println!(""); - print!("Generated bytecode: [ "); - for i in &bytecode { - print!("{i:02x} "); - } - println!("]"); +mod assembler; + +pub use addressing::{CharAddress, SourceLocation, BytecodeLocation}; +pub use syntactic_token::{SyntacticToken, SyntacticTokenType}; +pub use semantic_token::{SemanticToken, SemanticTokenType, LabelDefinition, MacroDefinition}; +pub use error::Error; +pub use tokenizer::TokenIterator; +pub use assembler::Assembler; + +pub fn assemble(source_code: &str) -> (Vec<u8>, Vec<SemanticToken>) { + let mut assembler = Assembler::new(); + assembler.tokenise_source(source_code); + assembler.resolve_references(); + assembler.generate_bytecode() } |