Added detection of cyclic macro references, and made assembler binary usable

7 files changed, 403 insertions, 382 deletions

diff --git a/src/assembler.rs b/src/assembler.rs
new file mode 100644
index 0000000..cb6b6f1
--- /dev/null
+++ b/src/assembler.rs
@@ -0,0 +1,273 @@
+use std::mem::take;
+use std::collections::hash_map::Entry;
+
+use SyntacticTokenType as Syn;
+use SemanticTokenType as Sem;
+use crate::*;
+
+use std::collections::HashMap;
+
+/// The inner value is the index of the token that defines this symbol.
+pub enum SymbolDefinition {
+    Macro(usize),
+    Label(usize),
+}
+
+pub struct Assembler {
+    /// The contents of the program as a list of syntactic tokens.
+    syntactic_tokens: Vec<SyntacticToken>,
+    /// The contents of the program as a list of semantic tokens.
+    semantic_tokens: Vec<SemanticToken>,
+    /// Map the name of each defined symbol to the index of the defining token.
+    symbol_definitions: HashMap<String, SymbolDefinition>,
+    /// Map each macro definition token index to a list of syntactic body tokens.
+    syntactic_macro_bodies: HashMap<usize, Vec<SyntacticToken>>,
+    /// Map each macro definition token index to a list of semantic body tokens.
+    semantic_macro_bodies: HashMap<usize, Vec<SemanticToken>>,
+}
+
+impl Assembler {
+    pub fn new() -> Self {
+        Self {
+            syntactic_tokens: Vec::new(),
+            semantic_tokens: Vec::new(),
+            symbol_definitions: HashMap::new(),
+            syntactic_macro_bodies: HashMap::new(),
+            semantic_macro_bodies: HashMap::new(),
+        }
+    }
+
+    pub fn tokenise_source(&mut self, source_code: &str) {
+        // The index of the current macro definition token
+        let mut macro_definition: Option<usize> = None;
+        let mut macro_definition_body_tokens: Vec<SyntacticToken> = Vec::new();
+
+        for mut token in TokenIterator::from_str(source_code) {
+            let next_index = self.syntactic_tokens.len();
+            if let Some(index) = macro_definition {
+                token.use_in_macro_body();
+                if token.is_macro_terminator() {
+                    // Commit the current macro definition
+                    macro_definition_body_tokens.push(token);
+                    self.syntactic_macro_bodies.insert(
+                        index, take(&mut macro_definition_body_tokens));
+                    macro_definition = None;
+                } else {
+                    macro_definition_body_tokens.push(token);
+                }
+            } else {
+                if let Syn::MacroDefinition(ref name) = token.r#type {
+                    macro_definition = Some(next_index);
+                    match self.symbol_definitions.entry(name.to_string()) {
+                        Entry::Occupied(_) => {token.set_error(Error::DuplicateDefinition);}
+                        Entry::Vacant(v) => {v.insert(SymbolDefinition::Macro(next_index));}
+                    }
+                } else if let Syn::LabelDefinition(ref name) = token.r#type {
+                    match self.symbol_definitions.entry(name.to_string()) {
+                        Entry::Occupied(_) => {token.set_error(Error::DuplicateDefinition);}
+                        Entry::Vacant(v) => {v.insert(SymbolDefinition::Label(next_index));}
+                    }
+                } else if token.is_macro_terminator() {
+                    token.set_error(Error::OrphanedMacroDefinitionTerminator);
+                }
+                self.syntactic_tokens.push(token);
+            }
+        }
+    }
+
+    pub fn resolve_references(&mut self) {
+        let syntactic_tokens = take(&mut self.syntactic_tokens);
+        let syntactic_token_count = syntactic_tokens.len();
+
+        for (index, syntactic_token) in syntactic_tokens.into_iter().enumerate() {
+            let semantic_token = self.convert_syn_token_to_sem_token(syntactic_token, index);
+            self.semantic_tokens.push(semantic_token);
+        }
+        assert_eq!(syntactic_token_count, self.semantic_tokens.len());
+
+        // Find all cyclic macros
+        let cyclic_macros: Vec<usize> = self.semantic_macro_bodies.keys().map(|i|*i).filter(
+            |i| !self.traverse_macro_definition(*i, 0)).collect();
+        // Replace each cyclic macro reference in a macro definition with an error
+        for body_tokens in &mut self.semantic_macro_bodies.values_mut() {
+            for body_token in body_tokens {
+                if let Sem::MacroReference(i) = body_token.r#type {
+                    if cyclic_macros.contains(&i) {
+                        let name = body_token.source_location.source.clone();
+                        body_token.r#type = Sem::Error(Syn::Reference(name), Error::CyclicMacroReference);
+                    }
+                }
+            }
+        }
+
+    }
+
+    /// Attempt to recursively traverse the body tokens of a macro definition, returning
+    /// false if the depth exceeds a preset maximum, and returning true otherwise.
+    fn traverse_macro_definition(&self, index: usize, level: usize) -> bool {
+        if level == 16 {
+            false
+        } else {
+            self.semantic_macro_bodies[&index].iter().all(
+                |token| if let Sem::MacroReference(i) = token.r#type {
+                    self.traverse_macro_definition(i, level+1)
+                } else {
+                    true
+                }
+            )
+        }
+    }
+
+    pub fn generate_bytecode(&mut self) -> (Vec<u8>, Vec<SemanticToken>) {
+        let mut bytecode: Vec<u8> = Vec::new();
+        // Map each label definition token index to the bytecode addresses of the references
+        let mut reference_addresses: HashMap<usize, Vec<u16>> = HashMap::new();
+        // Map each label and macro definition token to a list of reference token indices
+        let mut reference_tokens: HashMap<usize, Vec<usize>> = HashMap::new();
+
+        macro_rules! push_u8 {($v:expr) => {bytecode.push($v)};}
+        macro_rules! push_u16 {($v:expr) => {bytecode.extend_from_slice(&u16::to_be_bytes($v))};}
+        macro_rules! pad {($p:expr) => {bytecode.resize((bytecode.len() + $p as usize), 0)};}
+
+        let mut semantic_tokens = take(&mut self.semantic_tokens);
+
+        // Translate semantic tokens into bytecode
+        for (index, semantic_token) in semantic_tokens.iter_mut().enumerate() {
+            let start_addr = bytecode.len() as u16;
+            match &mut semantic_token.r#type {
+                Sem::LabelReference(i) => {
+                    reference_tokens.entry(*i).or_default().push(index);
+                    reference_addresses.entry(*i).or_default().push(start_addr);
+                    push_u16!(0);
+                }
+                Sem::MacroReference(i) => {
+                    reference_tokens.entry(*i).or_default().push(index);
+                    self.expand_macro_reference(*i, &mut bytecode, &mut reference_addresses);
+                }
+                Sem::LabelDefinition(def) => def.address=start_addr,
+                Sem::MacroDefinition(_) => (),
+
+                Sem::Padding(p) => pad!(*p),
+                Sem::ByteLiteral(b) => push_u8!(*b),
+                Sem::ShortLiteral(s) => push_u16!(*s),
+                Sem::Instruction(b) => push_u8!(*b),
+
+                Sem::MacroDefinitionTerminator => unreachable!(),
+                Sem::Comment => (),
+                Sem::Error(..) => (),
+            };
+            let end_addr = bytecode.len() as u16;
+            semantic_token.bytecode_location.start = start_addr;
+            semantic_token.bytecode_location.length = end_addr - start_addr;
+        }
+
+        // Fill each label reference with the address of the matching label definition
+        for (index, slots) in reference_addresses {
+            if let Sem::LabelDefinition(definition) = &semantic_tokens[index].r#type {
+                let [h,l] = definition.address.to_be_bytes();
+                for slot in slots {
+                    bytecode[slot as usize] = h;
+                    bytecode[slot.wrapping_add(1) as usize] = l;
+                }
+            } else { unreachable!() }
+        }
+
+        // Move references and macro body tokens into label and macro definition tokens
+        for (index, semantic_token) in semantic_tokens.iter_mut().enumerate() {
+            if let Sem::MacroDefinition(definition) = &mut semantic_token.r#type {
+                definition.body_tokens = self.semantic_macro_bodies.remove(&index).unwrap();
+                if let Some(references) = reference_tokens.remove(&index) {
+                    definition.references = references;
+                }
+            } else if let Sem::LabelDefinition(definition) = &mut semantic_token.r#type {
+                if let Some(references) = reference_tokens.remove(&index) {
+                    definition.references = references;
+                }
+            }
+        }
+        assert_eq!(reference_tokens.len(), 0);
+
+        // Remove trailing null bytes from the bytecode
+        if let Some(final_nonnull_byte) = bytecode.iter().rposition(|b| *b != 0) {
+            let truncated_length = final_nonnull_byte + 1;
+            let removed_byte_count = bytecode.len() - truncated_length;
+            if removed_byte_count > 0 {
+                bytecode.truncate(truncated_length);
+            }
+        }
+
+        (bytecode, semantic_tokens)
+    }
+
+    fn convert_syn_token_to_sem_token(&mut self, mut syn_token: SyntacticToken, index: usize) -> SemanticToken {
+        SemanticToken {
+            r#type: {
+                if let Some(err) = syn_token.error {
+                    Sem::Error(syn_token.r#type, err)
+                } else {
+                    match syn_token.r#type {
+                        Syn::Reference(ref name) => {
+                            match self.symbol_definitions.get(name) {
+                                Some(SymbolDefinition::Macro(i)) => Sem::MacroReference(*i),
+                                Some(SymbolDefinition::Label(i)) => Sem::LabelReference(*i),
+                                None => Sem::Error(syn_token.r#type, Error::UnresolvedReference),
+                            }
+                        }
+                        Syn::LabelDefinition(name) => {Sem::LabelDefinition(LabelDefinition::new(name))},
+                        Syn::MacroDefinition(name) => {
+                            let mut sem_body_tokens = Vec::new();
+                            for syn_body_token in self.syntactic_macro_bodies.remove(&index).unwrap() {
+                                // Make the source location of the macro definition token span the entire definition
+                                if syn_body_token.is_macro_terminator() {
+                                    syn_token.source_location.end = syn_body_token.source_location.start;
+                                }
+                                let sem_body_token = self.convert_syn_token_to_sem_token(syn_body_token, 0);
+                                sem_body_tokens.push(sem_body_token);
+                            }
+                            self.semantic_macro_bodies.insert(index, sem_body_tokens);
+                            Sem::MacroDefinition(MacroDefinition::new(name))
+                        },
+                        Syn::MacroDefinitionTerminator => Sem::MacroDefinitionTerminator,
+                        Syn::Padding(v) => Sem::Padding(v),
+                        Syn::ByteLiteral(v) => Sem::ByteLiteral(v),
+                        Syn::ShortLiteral(v) => Sem::ShortLiteral(v),
+                        Syn::Instruction(v) => Sem::Instruction(v),
+                        Syn::Comment => Sem::Comment,
+                    }
+                }
+            },
+            source_location: syn_token.source_location,
+            bytecode_location: BytecodeLocation::zero(),
+        }
+    }
+
+    fn expand_macro_reference(&self, index: usize, bytecode: &mut Vec<u8>, reference_addresses: &mut HashMap<usize, Vec<u16>>) {
+        macro_rules! push_u8 {($v:expr) => {bytecode.push($v)};}
+        macro_rules! push_u16 {($v:expr) => {bytecode.extend_from_slice(&u16::to_be_bytes($v))};}
+        macro_rules! pad {($p:expr) => {bytecode.resize((bytecode.len() + $p as usize), 0)};}
+
+        for body_token in self.semantic_macro_bodies.get(&index).unwrap() {
+            let start_addr = bytecode.len() as u16;
+            match &body_token.r#type {
+                Sem::LabelReference(i) => {
+                    reference_addresses.entry(*i).or_default().push(start_addr);
+                    push_u16!(0u16);
+                },
+                Sem::MacroReference(i) => {
+                    self.expand_macro_reference(*i, bytecode, reference_addresses);
+                },
+                Sem::LabelDefinition(_) => unreachable!(),
+                Sem::MacroDefinition(_) => unreachable!(),
+
+                Sem::Padding(p) => pad!(*p),
+                Sem::ByteLiteral(b) => push_u8!(*b),
+                Sem::ShortLiteral(s) => push_u16!(*s),
+                Sem::Instruction(b) => push_u8!(*b),
+
+                Sem::MacroDefinitionTerminator => (),
+                Sem::Comment => (),
+                Sem::Error(..) => (),
+            };
+        }
+    }
+}
diff --git a/src/error.rs b/src/error.rs
index efbfc4f..8a6c0d6 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,9 +1,10 @@
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone)]
 pub enum Error {
     UnresolvedReference,
     DuplicateDefinition,
-    InvalidHexadecimalLiteral,
+    InvalidPaddingValue,
     InvalidTypeInMacroDefinition,
-    OrphanedMacroTerminator,
-    TokenPastEndOfProgram,
+    OrphanedMacroDefinitionTerminator,
+    CyclicMacroReference,
 }
+
diff --git a/src/lib.rs b/src/lib.rs
index 5d84600..a657354 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -3,330 +3,19 @@ mod syntactic_token;
 mod semantic_token;
 mod tokenizer;
 mod error;
-
-pub use addressing::*;
-pub use syntactic_token::*;
-pub use semantic_token::*;
-pub use error::*;
-pub use tokenizer::*;
-
-use std::collections::hash_map::{HashMap, Entry};
-use std::mem::take;
-
-// On Unicode support: Work with characters, not bytes. This will eventually be
-// used in Verdant and Doctrine, and it'd be nice to be able to support other languages.
-// The only reason to work with bytes over characters would be for a minor decrease in complexity.
-// Only support the assembly of files of up to 64kB. If assets need to be tacked on the end,
-// it can be done by another program. The VM will only be able to access the first 64kB of a file anyway.
-// Treat \t as a space, have it be a single character.
-
-// First, turn the program source code into a vector of SyntacticTokens. These
-// each contain a SourceLocation, and the type and value of the token. Every single
-// non-whitespace character of the program needs to be wrapped in a SyntacticToken.
-// The program source code can be accurately reconstructed from this list of
-// SyntacticTokens, and when I write GRID, if the mouse is hovering over any point
-// in the program listing, I'll be able to determine the exact token that is being hovered.
-// For macros, hovering over any character belonging to a macro definition will
-// highlight the entire macro definition, and also the currently-hovered body token
-// if there is one. Clicking the body token will bring up more information.
-
-// The SyntacticTokens will be collected into a vector, with label and macro definition
-// being constructed as we go. Label definitions are easy, I only need to note down the
-// names of the labels in order to validate label references in a later step. If a label
-// name has already been defined, tag the token with an error. If a macro name has already
-// been defined, tag the token with an error.
-// Collect children into macro definitions. This makes sense.
-
-// Step 2 is to generate bytecode, converting SyntacticTokens into SemanticTokens.
-// Label and macro definitions need to contain a list of usizes to references.
-// Macro definitions need to contain the body tokens as SemanticTokens.
-// Label and macro references need to point to their parents.
-// Can I stream-convert tokens from Syntactic to Semantic?
-// Each SynToken gets converted to a SemToken? Yeah.
-
-// I want to change the parser to be a multi-stage struct thing, holding its own state.
-
-enum SymbolDefinition { Macro(usize), Label(usize) }
-
-pub fn parse(source_code: &str) {
-    use SyntacticTokenType as Syn;
-    use SemanticTokenType as Sem;
-
-    // ============================ STEP 1 ============================
-    // Convert the source code into a sorted vector of syntactic tokens and a
-    // map of symbol definitions.
-    // ================================================================
-    println!("[DEBUG] STEP 1: Parse source code into syntactic tokens");
-    let mut syntactic_tokens: Vec<SyntacticToken> = Vec::new();
-    let mut symbol_definitions: HashMap<String,SymbolDefinition> = HashMap::new();
-    let mut macro_bodies: HashMap<usize, Vec<SyntacticToken>> = HashMap::new();
-    let mut macro_definition: Option<usize> = None;
-    let mut macro_definition_body_tokens: Vec<SyntacticToken> = Vec::new();
-
-    for mut token in TokenIterator::from_str(source_code) {
-        if let Some(mdt) = macro_definition {
-            token.use_in_macro_body();
-            let terminate = token.is_macro_terminator();
-            macro_definition_body_tokens.push(token);
-            if terminate {
-                macro_bodies.insert(mdt, take(&mut macro_definition_body_tokens));
-                macro_definition = None;
-            }
-        } else {
-            if let Syn::MacroDefinition(ref name) = token.r#type {
-                macro_definition = Some(syntactic_tokens.len());
-                match symbol_definitions.entry(name.to_string()) {
-                    Entry::Occupied(_) => {token.set_error(Error::DuplicateDefinition);}
-                    Entry::Vacant(v) => {v.insert(SymbolDefinition::Macro(syntactic_tokens.len()));}
-                }
-            } else if let Syn::LabelDefinition(ref name) = token.r#type {
-                match symbol_definitions.entry(name.to_string()) {
-                    Entry::Occupied(_) => {token.set_error(Error::DuplicateDefinition);}
-                    Entry::Vacant(v) => {v.insert(SymbolDefinition::Label(syntactic_tokens.len()));}
-                }
-            } else if token.is_macro_terminator() {
-                token.set_error(Error::OrphanedMacroTerminator);
-            }
-            syntactic_tokens.push(token);
-        }
-    }
-
-
-    // ============================ STEP 2 ============================
-    // Convert syntactic tokens into semantic tokens, resolving label and macro
-    // references in the process.
-    // ================================================================
-    println!("[DEBUG] STEP 2: Resolve label and macro references");
-    let syntactic_token_count = syntactic_tokens.len();
-    let mut semantic_tokens = Vec::new();
-    let mut semantic_macro_bodies: HashMap<usize, Vec<SemanticToken>> = HashMap::new();
-
-    for (i, mut syn_token) in syntactic_tokens.into_iter().enumerate() {
-        let sem_token_type = if let Some(err) = syn_token.error {
-            // Translate over any existing syntax errors
-            Sem::Error(syn_token.r#type, err)
-        } else {
-            match syn_token.r#type {
-                Syn::Reference(ref name) => {
-                    match symbol_definitions.get(name) {
-                        Some(SymbolDefinition::Macro(addr)) => Sem::MacroReference(*addr),
-                        Some(SymbolDefinition::Label(addr)) => Sem::LabelReference(*addr),
-                        None => Sem::Error(syn_token.r#type, Error::UnresolvedReference),
-                    }
-                }
-                Syn::LabelDefinition(name) => {
-                    let label_definition = LabelDefinition {
-                        name, address: 0, references: Vec::new() };
-                    Sem::LabelDefinition(label_definition)
-                }
-                Syn::MacroDefinition(name) => {
-                    let mut sem_body_tokens = Vec::new();
-                    // Iterate over every token in the body of the macro definition,
-                    // converting each one to a semantic token.
-                    for syn_body_token in macro_bodies.remove(&i).unwrap() {
-                        let sem_body_token_type = if let Some(err) = syn_body_token.error {
-                            // Translate over any existing syntax errors
-                            Sem::Error(syn_body_token.r#type, err)
-                        } else {
-                            match syn_body_token.r#type {
-                                Syn::Reference(ref name) => match symbol_definitions.get(name) {
-                                    Some(SymbolDefinition::Macro(addr)) => Sem::MacroReference(*addr),
-                                    Some(SymbolDefinition::Label(addr)) => Sem::LabelReference(*addr),
-                                    None => Sem::Error(syn_body_token.r#type, Error::UnresolvedReference),
-                                },
-
-                                Syn::LabelDefinition(_) => unreachable!(),
-                                Syn::MacroDefinition(_) => unreachable!(),
-                                Syn::MacroTerminator => {
-                                    syn_token.source_location.end =
-                                        syn_body_token.source_location.end;
-                                    Sem::MacroTerminator
-                                },
-
-                                Syn::Pad(v) => Sem::Pad(v),
-                                Syn::Byte(v) => Sem::Byte(v),
-                                Syn::Short(v) => Sem::Short(v),
-                                Syn::Instruction(v) => Sem::Instruction(v),
-
-                                Syn::Comment => Sem::Comment,
-                            }
-                        };
-                        let sem_body_token = SemanticToken {
-                            r#type: sem_body_token_type,
-                            source_location: syn_body_token.source_location,
-                            bytecode_location: BytecodeLocation::zero(),
-                        };
-                        sem_body_tokens.push(sem_body_token);
-                    }
-                    semantic_macro_bodies.insert(i, sem_body_tokens);
-                    let macro_definition = MacroDefinition {
-                        name, body_tokens: Vec::new(), references: Vec::new() };
-                    Sem::MacroDefinition(macro_definition)
-                }
-                Syn::MacroTerminator => unreachable!(),
-
-                Syn::Pad(v) => Sem::Pad(v),
-                Syn::Byte(v) => Sem::Byte(v),
-                Syn::Short(v) => Sem::Short(v),
-                Syn::Instruction(v) => Sem::Instruction(v),
-
-                Syn::Comment => Sem::Comment,
-            }
-        };
-        let sem_token = SemanticToken {
-            r#type: sem_token_type,
-            source_location: syn_token.source_location,
-            bytecode_location: BytecodeLocation::zero(),
-        };
-        semantic_tokens.push(sem_token);
-    }
-    assert_eq!(syntactic_token_count, semantic_tokens.len());
-
-
-    // ============================ STEP 3 ============================
-    // Iterate over each semantic token, generating bytecode.
-    // ================================================================
-    println!("[DEBUG] STEP 3: Generate bytecode");
-    let mut bytecode: Vec<u8> = Vec::new();
-    // Map each label token to a list of bytecode addresses to populate
-    let mut label_reference_addresses: HashMap<usize, Vec<u16>> = HashMap::new();
-    // Map each label or macro definition token to a list of reference token pointers
-    let mut references: HashMap<usize, Vec<usize>> = HashMap::new();
-
-    macro_rules! addr {() => {bytecode.len() as u16};}
-    macro_rules! push_u8 {($v:expr) => {bytecode.push($v); 1};}
-    macro_rules! push_u16 {($v:expr) => {bytecode.extend_from_slice(&$v.to_be_bytes()); 2};}
-    macro_rules! pad {($p:expr) => {bytecode.resize(bytecode.len() + $p as usize, 0); $p as u16};}
-
-    for (i, sem_token) in semantic_tokens.iter_mut().enumerate() {
-        let start_addr = addr!();
-        let byte_length: u16 = match &mut sem_token.r#type {
-            Sem::LabelReference(addr) => {
-                references.entry(*addr).or_default().push(i);
-                label_reference_addresses.entry(*addr).or_default().push(addr!());
-                push_u16!(0u16); 2
-            },
-            Sem::MacroReference(addr) => {
-                references.entry(*addr).or_default().push(i);
-                let mut macro_byte_length: u16 = 0;
-                for body_token in semantic_macro_bodies.get(addr).unwrap() {
-                    macro_byte_length += match &body_token.r#type {
-                        Sem::LabelReference(addr) => {
-                            label_reference_addresses.entry(*addr).or_default().push(addr!());
-                            push_u16!(0u16); 2
-                        },
-                        Sem::MacroReference(_) => todo!(),
-
-                        Sem::LabelDefinition(_) => unreachable!(),
-                        Sem::MacroDefinition(_) => unreachable!(),
-
-                        Sem::Pad(p) => { pad!(*p); *p },
-                        Sem::Byte(b) => { push_u8!(*b); 1 },
-                        Sem::Short(s) => { push_u16!(*s); 2 },
-                        Sem::Instruction(b) => { push_u8!(*b); 1 },
-
-                        Sem::MacroTerminator => 0,
-                        Sem::Comment => 0,
-                        Sem::Error(..) => 0,
-                    };
-                }
-                macro_byte_length
-            },
-
-            Sem::LabelDefinition(definition) => {definition.address=addr!(); 1},
-            Sem::MacroDefinition(_) => 0,
-
-            Sem::Pad(p) => { pad!(*p); *p },
-            Sem::Byte(b) => { push_u8!(*b); 1 },
-            Sem::Short(s) => { push_u16!(*s); 2 },
-            Sem::Instruction(b) => { push_u8!(*b); 1 },
-
-            Sem::MacroTerminator => unreachable!(),
-            Sem::Comment => 0,
-            Sem::Error(..) => 0,
-        };
-        sem_token.bytecode_location.start = start_addr;
-        sem_token.bytecode_location.length = byte_length;
-    }
-
-
-    // ============================ STEP 4 ============================
-    // Fill in addresses for label references.
-    // ================================================================
-    println!("[DEBUG] STEP 4: Fill in values for label references");
-    for (label_i, slots) in label_reference_addresses.iter() {
-        if let Sem::LabelDefinition(LabelDefinition { address, .. }) = semantic_tokens[*label_i].r#type {
-            let [h,l] = address.to_be_bytes();
-            for slot in slots {
-                bytecode[*slot as usize] = h;
-                bytecode[slot.wrapping_add(1) as usize] = l;
-            }
-        } else {
-            unreachable!()
-        }
-    }
-
-    // ============================ STEP 5 ============================
-    // Move references and macro body tokens into label and macro definitions.
-    // ================================================================
-    println!("[DEBUG] STEP 5: Move information into label and macro definition tokens");
-    for (i, token) in semantic_tokens.iter_mut().enumerate() {
-        if let Sem::MacroDefinition(macro_definition) = &mut token.r#type {
-            macro_definition.body_tokens = semantic_macro_bodies.remove(&i).unwrap();
-            if let Some(macro_references) = references.remove(&i) {
-                macro_definition.references = macro_references;
-            }
-        } else if let Sem::LabelDefinition(label_definition) = &mut token.r#type {
-            if let Some(label_references) = references.remove(&i) {
-                label_definition.references = label_references;
-            }
-        }
-    }
-    assert_eq!(references.len(), 0);
-
-
-    // ============================ STEP 6 ============================
-    // Remove trailing null-bytes from the bytecode.
-    // ================================================================
-    println!("[DEBUG] STEP 6: Trim trailing null bytes");
-    if let Some(final_nonnull_byte) = bytecode.iter().rposition(|b| *b != 0) {
-        let truncated_length = final_nonnull_byte + 1;
-        let removed_byte_count = bytecode.len() - truncated_length;
-        if removed_byte_count > 0 {
-            println!("[INFO] Removed {removed_byte_count} trailing null bytes from assembled bytecode");
-            bytecode.truncate(truncated_length);
-        }
-    }
-
-
-    for token in &semantic_tokens {
-        if let Sem::MacroDefinition(macro_definition) = &token.r#type {
-            for body_token in &macro_definition.body_tokens {
-                if let Sem::Error(_, err) = body_token.r#type {
-                    println!("[ERROR] (in macro '{}') {err:?} at {}:{}..{}:{}",
-                        macro_definition.name,
-                        body_token.source_location.start.line,
-                        body_token.source_location.start.column,
-                        body_token.source_location.end.line,
-                        body_token.source_location.end.column,
-                    )
-                }
-            }
-        } else if let Sem::Error(_, err) = token.r#type {
-            println!("[ERROR {}:{}-{}:{}] {err:?}",
-                token.source_location.start.line,
-                token.source_location.start.column,
-                token.source_location.end.line,
-                token.source_location.end.column,
-            )
-        }
-    }
-
-    println!("");
-    print!("Generated bytecode: [ ");
-    for i in &bytecode {
-        print!("{i:02x} ");
-    }
-    println!("]");
+mod assembler;
+
+pub use addressing::{CharAddress, SourceLocation, BytecodeLocation};
+pub use syntactic_token::{SyntacticToken, SyntacticTokenType};
+pub use semantic_token::{SemanticToken, SemanticTokenType, LabelDefinition, MacroDefinition};
+pub use error::Error;
+pub use tokenizer::TokenIterator;
+pub use assembler::Assembler;
+
+pub fn assemble(source_code: &str) -> (Vec<u8>, Vec<SemanticToken>) {
+    let mut assembler = Assembler::new();
+    assembler.tokenise_source(source_code);
+    assembler.resolve_references();
+    assembler.generate_bytecode()
 }
 
diff --git a/src/main.rs b/src/main.rs
index 82bd92d..c7d3590 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,41 +1,33 @@
-const SOURCE:&'static str =
-// "
-// %SCREEN-SHUNT 00;
-// %SCREEN-DRAW 00;
-
-// @draw_horizontal_line  ( len* clr -- )
-// (1)  PSHr                       ( len*    | clr )
-// (4)  PSH:41 STD:SCREEN-SHUNT    ( len*    | clr )
-//       &loop
-// (2)   SKDr:SCREEN-DRAW          ( len*    | clr )
-// (4)   DEC* JKC*:~loop           ( len*    | clr )
-// (2)  STD:SCREEN-SHUNT
-// (3)  POP POPr JMPr*
-// ";
-
-"
-%RED 1234;
-%GREEN 5678 @test;
-%BLUE 9ABC;
-
-@start
-    RED
-    start
-    GREEN
-    BLUE
-    $4
-    @end
-";
-
+use std::io::{Read, Write};
 use bedrock_asm::*;
 
 fn main() {
-    println!("------- PROGRAM START -------");
-    for line in SOURCE.lines() {
-        println!("{line}");
-    }
-    println!("-------- PROGRAM END --------");
-    println!();
+    // Read source code from standard input
+    let mut source_code = String::new();
+    let mut stdin = std::io::stdin().lock();
+    if let Err(err) = stdin.read_to_string(&mut source_code) {
+        eprintln!("Could not read from standard input, quitting.");
+        eprintln!("({err:?})");
+        std::process::exit(1);
+    };
+
+    let (bytecode, tokens) = assemble(&source_code);
+    for token in &tokens {
+        token.print_error(&source_code); }
+    eprintln!("Assembled program in {} bytes.", bytecode.len());
+    let bytecode_len = bytecode.len();
 
-    parse(SOURCE);
+    // Write bytecode to standard output
+    let mut stdout = std::io::stdout().lock();
+    match stdout.write(&bytecode) {
+        Ok(len) => if len != bytecode_len {
+            eprintln!("Only wrote {len} of {bytecode_len} bytes")
+        }
+        Err(err) => {
+            eprintln!("Could not write to standard output, quitting.");
+            eprintln!("({err:?})");
+            std::process::exit(1);
+        }
+    }
 }
+
diff --git a/src/semantic_token.rs b/src/semantic_token.rs
index cac82a9..3d08b25 100644
--- a/src/semantic_token.rs
+++ b/src/semantic_token.rs
@@ -7,12 +7,12 @@ pub enum SemanticTokenType {
     LabelDefinition(LabelDefinition),
     MacroDefinition(MacroDefinition),
 
-    Pad(u16),
-    Byte(u8),
-    Short(u16),
+    Padding(u16),
+    ByteLiteral(u8),
+    ShortLiteral(u16),
     Instruction(u8),
 
-    MacroTerminator,
+    MacroDefinitionTerminator,
     Comment,
     Error(SyntacticTokenType, Error),
 }
@@ -23,16 +23,82 @@ pub struct SemanticToken {
     pub bytecode_location: BytecodeLocation,
 }
 
+impl SemanticToken {
+    pub fn print_error(&self, source_code: &str) {
+        macro_rules! red {()=>{eprint!("\x1b[31m")};}
+        macro_rules! normal {()=>{eprint!("\x1b[0m")};}
+
+        if let SemanticTokenType::Error(token, error) = &self.r#type {
+            red!(); eprint!("[ERROR] "); normal!();
+            let source = &self.source_location.source;
+            match error {
+                Error::UnresolvedReference => {
+                    eprintln!("Unresolved reference, no label or macro has been defined with the name '{source}'") }
+                Error::DuplicateDefinition => {
+                    eprintln!("Duplicate definition, a label or macro has already been defined with the name '{source}'") }
+                Error::OrphanedMacroDefinitionTerminator => {
+                    eprintln!("Unmatched macro definition terminator, no macro definition is in progress") }
+                Error::InvalidPaddingValue => {
+                    eprintln!("Invalid value for padding, the value must be at least one and at most four hexadecimal characters") }
+                Error::CyclicMacroReference => {
+                    eprintln!("Cyclic macro reference, this macro reference contains a reference to the macro being defined") }
+                Error::InvalidTypeInMacroDefinition => {
+                    let name = match token {
+                        SyntacticTokenType::Reference(_) => "references",
+                        SyntacticTokenType::LabelDefinition(_) => "label definitions",
+                        SyntacticTokenType::MacroDefinition(_) => "macro definitions",
+                        SyntacticTokenType::MacroDefinitionTerminator => "macro definition terminators",
+                        SyntacticTokenType::Padding(_) => "padding",
+                        SyntacticTokenType::ByteLiteral(_) => "byte literals",
+                        SyntacticTokenType::ShortLiteral(_) => "short literals",
+                        SyntacticTokenType::Instruction(_) => "instructions",
+                        SyntacticTokenType::Comment => "comments",
+                    };
+                    eprintln!("Invalid token in macro definition, macro definitions are not allowed to contain {name}") }
+            }
+            let line = source_code.split('\n').nth(self.source_location.start.line).unwrap();
+            eprint!("{:>5} ", self.source_location.start.line+1);
+            red!(); eprint!("| "); normal!();
+            for (i, c) in line.chars().enumerate() {
+                if i == self.source_location.start.column { red!() }
+                eprint!("{c}");
+                if i == self.source_location.end.column { normal!() }
+            }
+            eprintln!(); red!(); eprint!("      | ");
+            for i in 0..=self.source_location.end.column {
+                if i < self.source_location.start.column { eprint!(" ") } else { eprint!("^") };
+            }
+            normal!(); eprintln!();
+        }
+        if let SemanticTokenType::MacroDefinition(definition) = &self.r#type {
+            for token in &definition.body_tokens {
+                token.print_error(source_code);
+            }
+        }
+    }
+}
+
 pub struct LabelDefinition {
     pub name: String,
     pub address: u16,
     /// A list of pointers to label reference tokens
     pub references: Vec<usize>,
 }
+impl LabelDefinition {
+    pub fn new(name: String) -> Self {
+        Self { name, address:0, references:Vec::new() }
+    }
+}
+
 pub struct MacroDefinition {
     pub name: String,
     pub body_tokens: Vec<SemanticToken>,
     /// A list of pointers to macro reference tokens
     pub references: Vec<usize>,
 }
+impl MacroDefinition {
+    pub fn new(name: String) -> Self {
+        Self { name, body_tokens:Vec::new(), references:Vec::new() }
+    }
+}
 
diff --git a/src/syntactic_token.rs b/src/syntactic_token.rs
index ee473e2..4a50e8a 100644
--- a/src/syntactic_token.rs
+++ b/src/syntactic_token.rs
@@ -5,16 +5,18 @@ pub enum SyntacticTokenType {
 
     LabelDefinition(String),
     MacroDefinition(String),
-    MacroTerminator,
+    MacroDefinitionTerminator,
 
-    Pad(u16),
-    Byte(u8),
-    Short(u16),
+    Padding(u16),
+    ByteLiteral(u8),
+    ShortLiteral(u16),
     Instruction(u8),
 
     Comment,
 }
 
+
+
 pub struct SyntacticToken {
     pub r#type: SyntacticTokenType,
     pub source_location: SourceLocation,
@@ -32,12 +34,10 @@ impl SyntacticToken {
             _ => (),
         };
     }
-
     pub fn set_error(&mut self, error: Error) {
         self.error = Some(error);
     }
-
     pub fn is_macro_terminator(&self) -> bool {
-        if let SyntacticTokenType::MacroTerminator = self.r#type {true} else {false}
+        if let SyntacticTokenType::MacroDefinitionTerminator = self.r#type {true} else {false}
     }
 }
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index b68cc14..508daee 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -106,9 +106,9 @@ impl Iterator for TokenIterator {
         let full = take(&mut self.source);
         let suffix = take(&mut self.suffix);
         let mut error = None;
-        let mut parse_hex_lit = |v| {
+        let mut parse_padding_value = |v| {
             parse_short(v).or_else(|| {
-                error = Some(Error::InvalidHexadecimalLiteral); Some(0)
+                error = Some(Error::InvalidPaddingValue); Some(0)
             }).unwrap()
         };
 
@@ -116,13 +116,13 @@ impl Iterator for TokenIterator {
             '(' => { SyntacticTokenType::Comment }
             '@' => { SyntacticTokenType::LabelDefinition({self.label=suffix.clone(); suffix}) }
             '&' => { SyntacticTokenType::LabelDefinition(format!("{}/{}", self.label, suffix)) }
-            '$' => { SyntacticTokenType::Pad(parse_hex_lit(&suffix)) }
+            '$' => { SyntacticTokenType::Padding(parse_padding_value(&suffix)) }
             '~' => { SyntacticTokenType::Reference(format!("{}/{}", self.label, suffix)) }
             '%' => { SyntacticTokenType::MacroDefinition(suffix) }
             _ => {
-                if ";" == &full                                    { SyntacticTokenType::MacroTerminator }
-                else if let Some(value) = parse_byte_lit(&full)    { SyntacticTokenType::Byte(value) }
-                else if let Some(value) = parse_short_lit(&full)   { SyntacticTokenType::Short(value) }
+                if ";" == &full                                    { SyntacticTokenType::MacroDefinitionTerminator }
+                else if let Some(value) = parse_byte_lit(&full)    { SyntacticTokenType::ByteLiteral(value) }
+                else if let Some(value) = parse_short_lit(&full)   { SyntacticTokenType::ShortLiteral(value) }
                 else if let Some(value) = parse_instruction(&full) { SyntacticTokenType::Instruction(value) }
                 else                                               { SyntacticTokenType::Reference(full.clone()) }
             }