diff options
| -rw-r--r-- | src/tokenizer.rs | 38 |
1 files changed, 31 insertions, 7 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 508daee..2639453 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -13,6 +13,9 @@ pub struct TokenIterator { skip_whitespace: bool, /// The name of the most recently defined label. label: String, + /// If true, each individual character is tokenised as a ByteLiteral + parse_string_literal: bool, + /// The address of the first character of the current token. start: CharAddress, @@ -34,6 +37,7 @@ impl TokenIterator { i: 0, addr: CharAddress::zero(), skip_whitespace: true, + parse_string_literal: false, label: String::new(), start: CharAddress::zero(), end: CharAddress::zero(), @@ -60,11 +64,15 @@ impl TokenIterator { } /// Mark the current character as being the first character of a new token. fn mark_start(&mut self, c:char) { - self.start=self.addr; - self.end=self.addr; - self.prefix=c; - self.source.push(c); - self.skip_whitespace=false; + if c == '"' { + self.parse_string_literal = true; + } else { + self.start=self.addr; + self.end=self.addr; + self.prefix=c; + self.source.push(c); + self.skip_whitespace=false; + } self.next(c); } } @@ -80,6 +88,22 @@ impl Iterator for TokenIterator { // Iterate over source characters until a full token is read while let Some(c) = self.chars.get(self.i) { let c = *c; + // Parse individual characters from a string literal + if self.parse_string_literal { + if c == '"' { + self.parse_string_literal=false; + self.next(c); + continue + } else { + self.next(c); + return Some(SyntacticToken { + r#type: SyntacticTokenType::ByteLiteral(c as u8), + source_location: SourceLocation { + source: c.to_string(), start:self.addr, end:self.addr }, + error: None, + }); + } + } // Intercept comments if is_comment { self.push(c); if c == ')' { break; } else { continue }; } @@ -94,8 +118,8 @@ impl Iterator for TokenIterator { (false, false) => self.push(c), // c is a character of the token (true, false) => break, // c is trailing whitespace } - // Allow literal values to be attached to the end of the previous token - if self.source.len() > 0 && c == ':' { break; } + // Allow literal values to be chained to the end of the previous token + if self.source.len() > 0 && c == ':' { break } } // If no source characters were grabbed then we have read through the entire source file |