path: root/src/lib.rs

mod addressing;
mod syntactic_token;
mod semantic_token;
mod tokenizer;
mod error;

pub use addressing::*;
pub use syntactic_token::*;
pub use semantic_token::*;
pub use error::*;
pub use tokenizer::*;

use std::collections::hash_map::{HashMap, Entry};
use std::mem::take;

// On Unicode support: Work with characters, not bytes. This will eventually be
// used in Verdant and Doctrine, and it'd be nice to be able to support other languages.
// The only reason to work with bytes over characters would be for a minor decrease in complexity.
// Only support the assembly of files of up to 64kB. If assets need to be tacked on the end,
// it can be done by another program. The VM will only be able to access the first 64kB of a file anyway.
// Treat \t as a space, have it be a single character.

// First, turn the program source code into a vector of SyntacticTokens. These
// each contain a SourceLocation, and the type and value of the token. Every single
// non-whitespace character of the program needs to be wrapped in a SyntacticToken.
// The program source code can be accurately reconstructed from this list of
// SyntacticTokens, and when I write GRID, if the mouse is hovering over any point
// in the program listing, I'll be able to determine the exact token that is being hovered.
// For macros, hovering over any character belonging to a macro definition will
// highlight the entire macro definition, and also the currently-hovered body token
// if there is one. Clicking the body token will bring up more information.

// The SyntacticTokens will be collected into a vector, with label and macro definition
// being constructed as we go. Label definitions are easy, I only need to note down the
// names of the labels in order to validate label references in a later step. If a label
// name has already been defined, tag the token with an error. If a macro name has already
// been defined, tag the token with an error.
// Collect children into macro definitions. This makes sense.

// Step 2 is to generate bytecode, converting SyntacticTokens into SemanticTokens.
// Label and macro definitions need to contain a list of usizes to references.
// Macro definitions need to contain the body tokens as SemanticTokens.
// Label and macro references need to point to their parents.
// Can I stream-convert tokens from Syntactic to Semantic?
// Each SynToken gets converted to a SemToken? Yeah.

// I want to change the parser to be a multi-stage struct thing, holding its own state.

enum SymbolDefinition { Macro(usize), Label(usize) }

pub fn parse(source_code: &str) {
    use SyntacticTokenType as Syn;
    use SemanticTokenType as Sem;

    // ============================ STEP 1 ============================
    // Convert the source code into a sorted vector of syntactic tokens and a
    // map of symbol definitions.
    // ================================================================
    println!("[DEBUG] STEP 1: Parse source code into syntactic tokens");
    let mut syntactic_tokens: Vec<SyntacticToken> = Vec::new();
    let mut symbol_definitions: HashMap<String,SymbolDefinition> = HashMap::new();
    let mut macro_bodies: HashMap<usize, Vec<SyntacticToken>> = HashMap::new();
    let mut macro_definition: Option<usize> = None;
    let mut macro_definition_body_tokens: Vec<SyntacticToken> = Vec::new();

    for mut token in TokenIterator::from_str(source_code) {
        if let Some(mdt) = macro_definition {
            token.use_in_macro_body();
            let terminate = token.is_macro_terminator();
            macro_definition_body_tokens.push(token);
            if terminate {
                macro_bodies.insert(mdt, take(&mut macro_definition_body_tokens));
                macro_definition = None;
            }
        } else {
            if let Syn::MacroDefinition(ref name) = token.r#type {
                macro_definition = Some(syntactic_tokens.len());
                match symbol_definitions.entry(name.to_string()) {
                    Entry::Occupied(_) => {token.set_error(Error::DuplicateDefinition);}
                    Entry::Vacant(v) => {v.insert(SymbolDefinition::Macro(syntactic_tokens.len()));}
                }
            } else if let Syn::LabelDefinition(ref name) = token.r#type {
                match symbol_definitions.entry(name.to_string()) {
                    Entry::Occupied(_) => {token.set_error(Error::DuplicateDefinition);}
                    Entry::Vacant(v) => {v.insert(SymbolDefinition::Label(syntactic_tokens.len()));}
                }
            } else if token.is_macro_terminator() {
                token.set_error(Error::OrphanedMacroTerminator);
            }
            syntactic_tokens.push(token);
        }
    }


    // ============================ STEP 2 ============================
    // Convert syntactic tokens into semantic tokens, resolving label and macro
    // references in the process.
    // ================================================================
    println!("[DEBUG] STEP 2: Resolve label and macro references");
    let syntactic_token_count = syntactic_tokens.len();
    let mut semantic_tokens = Vec::new();
    let mut semantic_macro_bodies: HashMap<usize, Vec<SemanticToken>> = HashMap::new();

    for (i, mut syn_token) in syntactic_tokens.into_iter().enumerate() {
        let sem_token_type = if let Some(err) = syn_token.error {
            // Translate over any existing syntax errors
            Sem::Error(syn_token.r#type, err)
        } else {
            match syn_token.r#type {
                Syn::Reference(ref name) => {
                    match symbol_definitions.get(name) {
                        Some(SymbolDefinition::Macro(addr)) => Sem::MacroReference(*addr),
                        Some(SymbolDefinition::Label(addr)) => Sem::LabelReference(*addr),
                        None => Sem::Error(syn_token.r#type, Error::UnresolvedReference),
                    }
                }
                Syn::LabelDefinition(name) => {
                    let label_definition = LabelDefinition {
                        name, address: 0, references: Vec::new() };
                    Sem::LabelDefinition(label_definition)
                }
                Syn::MacroDefinition(name) => {
                    let mut sem_body_tokens = Vec::new();
                    // Iterate over every token in the body of the macro definition,
                    // converting each one to a semantic token.
                    for syn_body_token in macro_bodies.remove(&i).unwrap() {
                        let sem_body_token_type = if let Some(err) = syn_body_token.error {
                            // Translate over any existing syntax errors
                            Sem::Error(syn_body_token.r#type, err)
                        } else {
                            match syn_body_token.r#type {
                                Syn::Reference(ref name) => match symbol_definitions.get(name) {
                                    Some(SymbolDefinition::Macro(addr)) => Sem::MacroReference(*addr),
                                    Some(SymbolDefinition::Label(addr)) => Sem::LabelReference(*addr),
                                    None => Sem::Error(syn_body_token.r#type, Error::UnresolvedReference),
                                },

                                Syn::LabelDefinition(_) => unreachable!(),
                                Syn::MacroDefinition(_) => unreachable!(),
                                Syn::MacroTerminator => {
                                    syn_token.source_location.end =
                                        syn_body_token.source_location.end;
                                    Sem::MacroTerminator
                                },

                                Syn::Pad(v) => Sem::Pad(v),
                                Syn::Byte(v) => Sem::Byte(v),
                                Syn::Short(v) => Sem::Short(v),
                                Syn::Instruction(v) => Sem::Instruction(v),

                                Syn::Comment => Sem::Comment,
                            }
                        };
                        let sem_body_token = SemanticToken {
                            r#type: sem_body_token_type,
                            source_location: syn_body_token.source_location,
                            bytecode_location: BytecodeLocation::zero(),
                        };
                        sem_body_tokens.push(sem_body_token);
                    }
                    semantic_macro_bodies.insert(i, sem_body_tokens);
                    let macro_definition = MacroDefinition {
                        name, body_tokens: Vec::new(), references: Vec::new() };
                    Sem::MacroDefinition(macro_definition)
                }
                Syn::MacroTerminator => unreachable!(),

                Syn::Pad(v) => Sem::Pad(v),
                Syn::Byte(v) => Sem::Byte(v),
                Syn::Short(v) => Sem::Short(v),
                Syn::Instruction(v) => Sem::Instruction(v),

                Syn::Comment => Sem::Comment,
            }
        };
        let sem_token = SemanticToken {
            r#type: sem_token_type,
            source_location: syn_token.source_location,
            bytecode_location: BytecodeLocation::zero(),
        };
        semantic_tokens.push(sem_token);
    }
    assert_eq!(syntactic_token_count, semantic_tokens.len());


    // ============================ STEP 3 ============================
    // Iterate over each semantic token, generating bytecode.
    // ================================================================
    println!("[DEBUG] STEP 3: Generate bytecode");
    let mut bytecode: Vec<u8> = Vec::new();
    // Map each label token to a list of bytecode addresses to populate
    let mut label_reference_addresses: HashMap<usize, Vec<u16>> = HashMap::new();
    // Map each label or macro definition token to a list of reference token pointers
    let mut references: HashMap<usize, Vec<usize>> = HashMap::new();

    macro_rules! addr {() => {bytecode.len() as u16};}
    macro_rules! push_u8 {($v:expr) => {bytecode.push($v); 1};}
    macro_rules! push_u16 {($v:expr) => {bytecode.extend_from_slice(&$v.to_be_bytes()); 2};}
    macro_rules! pad {($p:expr) => {bytecode.resize(bytecode.len() + $p as usize, 0); $p as u16};}

    for (i, sem_token) in semantic_tokens.iter_mut().enumerate() {
        let start_addr = addr!();
        let byte_length: u16 = match &mut sem_token.r#type {
            Sem::LabelReference(addr) => {
                references.entry(*addr).or_default().push(i);
                label_reference_addresses.entry(*addr).or_default().push(addr!());
                push_u16!(0u16); 2
            },
            Sem::MacroReference(addr) => {
                references.entry(*addr).or_default().push(i);
                let mut macro_byte_length: u16 = 0;
                for body_token in semantic_macro_bodies.get(addr).unwrap() {
                    macro_byte_length += match &body_token.r#type {
                        Sem::LabelReference(addr) => {
                            label_reference_addresses.entry(*addr).or_default().push(addr!());
                            push_u16!(0u16); 2
                        },
                        Sem::MacroReference(_) => todo!(),

                        Sem::LabelDefinition(_) => unreachable!(),
                        Sem::MacroDefinition(_) => unreachable!(),

                        Sem::Pad(p) => { pad!(*p); *p },
                        Sem::Byte(b) => { push_u8!(*b); 1 },
                        Sem::Short(s) => { push_u16!(*s); 2 },
                        Sem::Instruction(b) => { push_u8!(*b); 1 },

                        Sem::MacroTerminator => 0,
                        Sem::Comment => 0,
                        Sem::Error(..) => 0,
                    };
                }
                macro_byte_length
            },

            Sem::LabelDefinition(definition) => {definition.address=addr!(); 1},
            Sem::MacroDefinition(_) => 0,

            Sem::Pad(p) => { pad!(*p); *p },
            Sem::Byte(b) => { push_u8!(*b); 1 },
            Sem::Short(s) => { push_u16!(*s); 2 },
            Sem::Instruction(b) => { push_u8!(*b); 1 },

            Sem::MacroTerminator => unreachable!(),
            Sem::Comment => 0,
            Sem::Error(..) => 0,
        };
        sem_token.bytecode_location.start = start_addr;
        sem_token.bytecode_location.length = byte_length;
    }


    // ============================ STEP 4 ============================
    // Fill in addresses for label references.
    // ================================================================
    println!("[DEBUG] STEP 4: Fill in values for label references");
    for (label_i, slots) in label_reference_addresses.iter() {
        if let Sem::LabelDefinition(LabelDefinition { address, .. }) = semantic_tokens[*label_i].r#type {
            let [h,l] = address.to_be_bytes();
            for slot in slots {
                bytecode[*slot as usize] = h;
                bytecode[slot.wrapping_add(1) as usize] = l;
            }
        } else {
            unreachable!()
        }
    }

    // ============================ STEP 5 ============================
    // Move references and macro body tokens into label and macro definitions.
    // ================================================================
    println!("[DEBUG] STEP 5: Move information into label and macro definition tokens");
    for (i, token) in semantic_tokens.iter_mut().enumerate() {
        if let Sem::MacroDefinition(macro_definition) = &mut token.r#type {
            macro_definition.body_tokens = semantic_macro_bodies.remove(&i).unwrap();
            if let Some(macro_references) = references.remove(&i) {
                macro_definition.references = macro_references;
            }
        } else if let Sem::LabelDefinition(label_definition) = &mut token.r#type {
            if let Some(label_references) = references.remove(&i) {
                label_definition.references = label_references;
            }
        }
    }
    assert_eq!(references.len(), 0);


    // ============================ STEP 6 ============================
    // Remove trailing null-bytes from the bytecode.
    // ================================================================
    println!("[DEBUG] STEP 6: Trim trailing null bytes");
    if let Some(final_nonnull_byte) = bytecode.iter().rposition(|b| *b != 0) {
        let truncated_length = final_nonnull_byte + 1;
        let removed_byte_count = bytecode.len() - truncated_length;
        if removed_byte_count > 0 {
            println!("[INFO] Removed {removed_byte_count} trailing null bytes from assembled bytecode");
            bytecode.truncate(truncated_length);
        }
    }


    for token in &semantic_tokens {
        if let Sem::MacroDefinition(macro_definition) = &token.r#type {
            for body_token in &macro_definition.body_tokens {
                if let Sem::Error(_, err) = body_token.r#type {
                    println!("[ERROR] (in macro '{}') {err:?} at {}:{}..{}:{}",
                        macro_definition.name,
                        body_token.source_location.start.line,
                        body_token.source_location.start.column,
                        body_token.source_location.end.line,
                        body_token.source_location.end.column,
                    )
                }
            }
        } else if let Sem::Error(_, err) = token.r#type {
            println!("[ERROR {}:{}-{}:{}] {err:?}",
                token.source_location.start.line,
                token.source_location.start.column,
                token.source_location.end.line,
                token.source_location.end.column,
            )
        }
    }

    println!("");
    print!("Generated bytecode: [ ");
    for i in &bytecode {
        print!("{i:02x} ");
    }
    println!("]");
}