Rewrite assembler

This is an almost complete rewrite of the entire assembler from the
ground up, with a different parsing strategy and a whole new symbol
resolution mechanism for automatically including library files.

The assembly syntax has also been slightly modified, with padding
tokens now being prefixed with '#' instead of '$', and a block-style
anonymous-label syntax which uses the '{' and '}' characters.

1 files changed, 0 insertions, 235 deletions

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
deleted file mode 100644
index 02bf490..0000000
--- a/src/tokenizer.rs
+++ /dev/null
@@ -1,235 +0,0 @@
-use std::mem::take;
-use crate::*;
-
-#[derive(PartialEq)]
-enum StringLiteral {
-    None,
-    Raw,
-    NullTerminated,
-}
-
-pub struct TokenIterator {
-    /// The characters that comprise the program souce code.
-    chars: Vec<char>,
-    /// The index of the next character to read.
-    i: usize,
-    /// The address of the next character to read.
-    addr: CharAddress,
-    /// If true, skip over any whitespace characters. If false, stop reading
-    /// when a whitespace character is encountered.
-    skip_whitespace: bool,
-    /// The name of the most recently defined label.
-    label: String,
-    /// If not None, each individual character will be tokenised as a ByteLiteral.
-    parse_string_literal: StringLiteral,
-
-
-    /// The address of the first character of the current token.
-    start: CharAddress,
-    /// The address of the final character of the current token.
-    end: CharAddress,
-    /// The entire current token.
-    source: String,
-    /// The first character of the current token.
-    prefix: char,
-    /// The second and remaining characters of the current token.
-    suffix: String,
-}
-
-impl TokenIterator {
-    /// Create an iterator from a string of program source code.
-    pub fn from_str(source_code: &str) -> Self {
-        Self {
-            chars: source_code.chars().collect(),
-            i: 0,
-            addr: CharAddress::zero(),
-            skip_whitespace: true,
-            parse_string_literal: StringLiteral::None,
-            label: String::new(),
-            start: CharAddress::zero(),
-            end: CharAddress::zero(),
-            source: String::new(),
-            prefix: ' ',
-            suffix: String::new(),
-        }
-    }
-    /// Append a character to the current token.
-    fn push(&mut self, c:char) {
-        self.end = self.addr;
-        self.source.push(c);
-        self.suffix.push(c);
-        self.next(c);
-    }
-    /// Move forward to the next source character.
-    fn next(&mut self, c: char) {
-        self.addr.column += 1;
-        self.i += 1;
-        if c == '\n' {
-            self.addr.column = 0;
-            self.addr.line += 1;
-        }
-    }
-    /// Mark the current character as being the first character of a new token.
-    fn mark_start(&mut self, c:char) {
-        if c == '"' {
-            self.parse_string_literal = StringLiteral::NullTerminated;
-        } else if c == '\'' {
-            self.parse_string_literal = StringLiteral::Raw;
-        } else {
-            self.start=self.addr;
-            self.end=self.addr;
-            self.prefix=c;
-            self.source.push(c);
-            self.skip_whitespace=false;
-        }
-        self.next(c);
-    }
-}
-
-impl Iterator for TokenIterator {
-    type Item = SyntacticToken;
-
-    fn next(&mut self) -> Option<SyntacticToken> {
-        // Initialise values before reading the next token
-        let mut is_comment = false;
-        self.skip_whitespace = true;
-
-        // Iterate over source characters until a full token is read
-        while let Some(c) = self.chars.get(self.i) {
-            let c = *c;
-            // Parse individual characters from a string literal
-            if self.parse_string_literal != StringLiteral::None {
-                if c == '"' && self.parse_string_literal == StringLiteral::NullTerminated  {
-                    self.parse_string_literal = StringLiteral::None;
-                    let token = SyntacticToken {
-                        r#type: SyntacticTokenType::ByteLiteral(0),
-                        source_location: SourceLocation {
-                            source: c.to_string(), start:self.addr, end:self.addr },
-                        error: None,
-                    };
-                    self.next(c);
-                    return Some(token);
-                } else if c == '\'' && self.parse_string_literal == StringLiteral::Raw  {
-                    self.parse_string_literal = StringLiteral::None;
-                    self.next(c);
-                    continue
-                } else {
-                    self.next(c);
-                    return Some(SyntacticToken {
-                        r#type: SyntacticTokenType::ByteLiteral(c as u8),
-                        source_location: SourceLocation {
-                            source: c.to_string(), start:self.addr, end:self.addr },
-                        error: None,
-                    });
-                }
-            }
-            // Intercept comments
-            if is_comment {
-                self.push(c); if c == ')' { break } else { continue }; }
-            else if self.skip_whitespace && c == '(' {
-                is_comment = true; self.mark_start(c); continue }
-
-            // Allow a semicolon at the end of a token to be handled as a separate token
-            if self.source.len() > 0 && c == ';' { break }
-            // Handle the current character
-            match (is_whitespace(c), self.skip_whitespace) {
-                (true, true) => self.next(c),        // c is the expected leading whitespace
-                (false, true) => self.mark_start(c), // c is the first character of the token
-                (false, false) => self.push(c),      // c is a character of the token
-                (true, false) => break,              // c is trailing whitespace
-            }
-            // Allow literal values to be chained to the end of the previous token
-            if self.source.len() > 0 && c == ':' { break }
-        }
-
-        // If no source characters were grabbed then we have read through the entire source file
-        if self.source.len() == 0 { return None; }
-        // Allow handling macro terminators and symbols of length 1 in the match expression
-        if self.suffix.len() == 0 { self.prefix = '\0'; }
-        // Consume the collected characters to be used in the match expression
-        let full = take(&mut self.source);
-        let suffix = take(&mut self.suffix);
-        let mut error = None;
-        let mut parse_padding_value = |v| {
-            parse_short(v).or_else(|| {
-                error = Some(Error::InvalidPaddingValue); Some(0)
-            }).unwrap()
-        };
-
-        let r#type = match self.prefix {
-            '(' => { SyntacticTokenType::Comment }
-            '@' => { SyntacticTokenType::LabelDefinition({self.label=suffix.clone(); suffix}) }
-            '&' => { SyntacticTokenType::LabelDefinition(format!("{}/{}", self.label, suffix)) }
-            '$' => { SyntacticTokenType::Padding(parse_padding_value(&suffix)) }
-            '~' => { SyntacticTokenType::Reference(format!("{}/{}", self.label, suffix)) }
-            '%' => if let Some(("", sublabel)) = suffix.split_once("~") {
-                SyntacticTokenType::MacroDefinition(format!("{}/{}", self.label, sublabel))
-            } else {
-                SyntacticTokenType::MacroDefinition(suffix)
-            }
-            _ => {
-                if ";" == &full                                    { SyntacticTokenType::MacroDefinitionTerminator }
-                else if let Some(value) = parse_byte_lit(&full)    { SyntacticTokenType::ByteLiteral(value) }
-                else if let Some(value) = parse_short_lit(&full)   { SyntacticTokenType::ShortLiteral(value) }
-                else if let Some(value) = parse_instruction(&full) { SyntacticTokenType::Instruction(value) }
-                else                                               { SyntacticTokenType::Reference(full.clone()) }
-            }
-        };
-        Some(SyntacticToken {
-            r#type,
-            source_location:SourceLocation::new(full,self.start,self.end),
-            error,
-        })
-    }
-}
-
-
-fn parse_byte_lit(token: &str) -> Option<u8> {
-    match token.len() { 2 => u8::from_str_radix(token, 16).ok(), _ => None } }
-fn parse_short_lit(token: &str) -> Option<u16> {
-    match token.len() { 4 => u16::from_str_radix(token, 16).ok(), _ => None } }
-fn parse_short(token: &str) -> Option<u16> {
-    match token.len() { 1..=4 => u16::from_str_radix(token, 16).ok(), _ => None } }
-fn is_whitespace(c: char) -> bool {
-    match c { ' '|'\t'|'\n'|'\r'|'['|']'|'(' =>true, _=>false } }
-fn parse_instruction(token: &str) -> Option<u8> {
-    Some(match token {
-        // Control operators
-        "HLT"=>0x00,"NOP" =>0x20,"DB1" =>0x40,"DB2"  =>0x60,"DB3" =>0x80,"DB4"  =>0xA0,"DB5"  =>0xC0,"DB6"   =>0xE0,
-        "JMP"=>0x01,"JMS" =>0x21,"JMP:"=>0x41,"JMS:" =>0x61,"JMPr"=>0x81,"JMSr" =>0xA1,"JMPr:"=>0xC1,"JMSr:" =>0xE1,
-        "JCN"=>0x02,"JCS" =>0x22,"JCN:"=>0x42,"JCS:" =>0x62,"JCNr"=>0x82,"JCSr" =>0xA2,"JCNr:"=>0xC2,"JCSr:" =>0xE2,
-        "JCK"=>0x03,"JCK*"=>0x23,"JCK:"=>0x43,"JCK*:"=>0x63,"JCKr"=>0x83,"JCKr*"=>0xA3,"JCKr:"=>0xC3,"JCKr*:"=>0xE3,
-        "LDA"=>0x04,"LDA*"=>0x24,"LDA:"=>0x44,"LDA*:"=>0x64,"LDAr"=>0x84,"LDAr*"=>0xA4,"LDAr:"=>0xC4,"LDAr*:"=>0xE4,
-        "STA"=>0x05,"STA*"=>0x25,"STA:"=>0x45,"STA*:"=>0x65,"STAr"=>0x85,"STAr*"=>0xA5,"STAr:"=>0xC5,"STAr*:"=>0xE5,
-        "LDD"=>0x06,"LDD*"=>0x26,"LDD:"=>0x46,"LDD*:"=>0x66,"LDDr"=>0x86,"LDDr*"=>0xA6,"LDDr:"=>0xC6,"LDDr*:"=>0xE6,
-        "STD"=>0x07,"STD*"=>0x27,"STD:"=>0x47,"STD*:"=>0x67,"STDr"=>0x87,"STDr*"=>0xA7,"STDr:"=>0xC7,"STDr*:"=>0xE7,
-        // Stack operators
-        "PSH"=>0x08,"PSH*"=>0x28,"PSH:"=>0x48,"PSH*:"=>0x68,"PSHr"=>0x88,"PSHr*"=>0xA8,"PSHr:"=>0xC8,"PSHr*:"=>0xE8,
-        "POP"=>0x09,"POP*"=>0x29,"POP:"=>0x49,"POP*:"=>0x69,"POPr"=>0x89,"POPr*"=>0xA9,"POPr:"=>0xC9,"POPr*:"=>0xE9,
-        "CPY"=>0x0A,"CPY*"=>0x2A,"CPY:"=>0x4A,"CPY*:"=>0x6A,"CPYr"=>0x8A,"CPYr*"=>0xAA,"CPYr:"=>0xCA,"CPYr*:"=>0xEA,
-        "SPL"=>0x0B,"SPL*"=>0x2B,"SPL:"=>0x4B,"SPL*:"=>0x6B,"SPLr"=>0x8B,"SPLr*"=>0xAB,"SPLr:"=>0xCB,"SPLr*:"=>0xEB,
-        "DUP"=>0x0C,"DUP*"=>0x2C,"DUP:"=>0x4C,"DUP*:"=>0x6C,"DUPr"=>0x8C,"DUPr*"=>0xAC,"DUPr:"=>0xCC,"DUPr*:"=>0xEC,
-        "OVR"=>0x0D,"OVR*"=>0x2D,"OVR:"=>0x4D,"OVR*:"=>0x6D,"OVRr"=>0x8D,"OVRr*"=>0xAD,"OVRr:"=>0xCD,"OVRr*:"=>0xED,
-        "SWP"=>0x0E,"SWP*"=>0x2E,"SWP:"=>0x4E,"SWP*:"=>0x6E,"SWPr"=>0x8E,"SWPr*"=>0xAE,"SWPr:"=>0xCE,"SWPr*:"=>0xEE,
-        "ROT"=>0x0F,"ROT*"=>0x2F,"ROT:"=>0x4F,"ROT*:"=>0x6F,"ROTr"=>0x8F,"ROTr*"=>0xAF,"ROTr:"=>0xCF,"ROTr*:"=>0xEF,
-        // Numeric operators
-        "ADD"=>0x10,"ADD*"=>0x30,"ADD:"=>0x50,"ADD*:"=>0x70,"ADDr"=>0x90,"ADDr*"=>0xB0,"ADDr:"=>0xD0,"ADDr*:"=>0xF0,
-        "SUB"=>0x11,"SUB*"=>0x31,"SUB:"=>0x51,"SUB*:"=>0x71,"SUBr"=>0x91,"SUBr*"=>0xB1,"SUBr:"=>0xD1,"SUBr*:"=>0xF1,
-        "INC"=>0x12,"INC*"=>0x32,"INC:"=>0x52,"INC*:"=>0x72,"INCr"=>0x92,"INCr*"=>0xB2,"INCr:"=>0xD2,"INCr*:"=>0xF2,
-        "DEC"=>0x13,"DEC*"=>0x33,"DEC:"=>0x53,"DEC*:"=>0x73,"DECr"=>0x93,"DECr*"=>0xB3,"DECr:"=>0xD3,"DECr*:"=>0xF3,
-        "LTH"=>0x14,"LTH*"=>0x34,"LTH:"=>0x54,"LTH*:"=>0x74,"LTHr"=>0x94,"LTHr*"=>0xB4,"LTHr:"=>0xD4,"LTHr*:"=>0xF4,
-        "GTH"=>0x15,"GTH*"=>0x35,"GTH:"=>0x55,"GTH*:"=>0x75,"GTHr"=>0x95,"GTHr*"=>0xB5,"GTHr:"=>0xD5,"GTHr*:"=>0xF5,
-        "EQU"=>0x16,"EQU*"=>0x36,"EQU:"=>0x56,"EQU*:"=>0x76,"EQUr"=>0x96,"EQUr*"=>0xB6,"EQUr:"=>0xD6,"EQUr*:"=>0xF6,
-        "NQK"=>0x17,"NQK*"=>0x37,"NQK:"=>0x57,"NQK*:"=>0x77,"NQKr"=>0x97,"NQKr*"=>0xB7,"NQKr:"=>0xD7,"NQKr*:"=>0xF7,
-        // Bitwise operators
-        "IOR"=>0x18,"IOR*"=>0x38,"IOR:"=>0x58,"IOR*:"=>0x78,"IORr"=>0x98,"IORr*"=>0xB8,"IORr:"=>0xD8,"IORr*:"=>0xF8,
-        "XOR"=>0x19,"XOR*"=>0x39,"XOR:"=>0x59,"XOR*:"=>0x79,"XORr"=>0x99,"XORr*"=>0xB9,"XORr:"=>0xD9,"XORr*:"=>0xF9,
-        "AND"=>0x1A,"AND*"=>0x3A,"AND:"=>0x5A,"AND*:"=>0x7A,"ANDr"=>0x9A,"ANDr*"=>0xBA,"ANDr:"=>0xDA,"ANDr*:"=>0xFA,
-        "NOT"=>0x1B,"NOT*"=>0x3B,"NOT:"=>0x5B,"NOT*:"=>0x7B,"NOTr"=>0x9B,"NOTr*"=>0xBB,"NOTr:"=>0xDB,"NOTr*:"=>0xFB,
-        "SHF"=>0x1C,"SHF*"=>0x3C,"SHF:"=>0x5C,"SHF*:"=>0x7C,"SHFr"=>0x9C,"SHFr*"=>0xBC,"SHFr:"=>0xDC,"SHFr*:"=>0xFC,
-        "SHC"=>0x1D,"SHC*"=>0x3D,"SHC:"=>0x5D,"SHC*:"=>0x7D,"SHCr"=>0x9D,"SHCr*"=>0xBD,"SHCr:"=>0xDD,"SHCr*:"=>0xFD,
-        "TAL"=>0x1E,"TAL*"=>0x3E,"TAL:"=>0x5E,"TAL*:"=>0x7E,"TALr"=>0x9E,"TALr*"=>0xBE,"TALr:"=>0xDE,"TALr*:"=>0xFE,
-        "REV"=>0x1F,"REV*"=>0x3F,"REV:"=>0x5F,"REV*:"=>0x7F,"REVr"=>0x9F,"REVr*"=>0xBF,"REVr:"=>0xDF,"REVr*:"=>0xFF,
-        _ => return None,
-    })
-}