1 files changed, 31 insertions, 7 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 508daee..2639453 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -13,6 +13,9 @@ pub struct TokenIterator {
     skip_whitespace: bool,
     /// The name of the most recently defined label.
     label: String,
+    /// If true, each individual character is tokenised as a ByteLiteral
+    parse_string_literal: bool,
+
 
     /// The address of the first character of the current token.
     start: CharAddress,
@@ -34,6 +37,7 @@ impl TokenIterator {
             i: 0,
             addr: CharAddress::zero(),
             skip_whitespace: true,
+            parse_string_literal: false,
             label: String::new(),
             start: CharAddress::zero(),
             end: CharAddress::zero(),
@@ -60,11 +64,15 @@ impl TokenIterator {
     }
     /// Mark the current character as being the first character of a new token.
     fn mark_start(&mut self, c:char) {
-        self.start=self.addr;
-        self.end=self.addr;
-        self.prefix=c;
-        self.source.push(c);
-        self.skip_whitespace=false;
+        if c == '"' {
+            self.parse_string_literal = true;
+        } else {
+            self.start=self.addr;
+            self.end=self.addr;
+            self.prefix=c;
+            self.source.push(c);
+            self.skip_whitespace=false;
+        }
         self.next(c);
     }
 }
@@ -80,6 +88,22 @@ impl Iterator for TokenIterator {
         // Iterate over source characters until a full token is read
         while let Some(c) = self.chars.get(self.i) {
             let c = *c;
+            // Parse individual characters from a string literal
+            if self.parse_string_literal {
+                if c == '"' {
+                    self.parse_string_literal=false;
+                    self.next(c);
+                    continue
+                } else {
+                    self.next(c);
+                    return Some(SyntacticToken {
+                        r#type: SyntacticTokenType::ByteLiteral(c as u8),
+                        source_location: SourceLocation {
+                            source: c.to_string(), start:self.addr, end:self.addr },
+                        error: None,
+                    });
+                }
+            }
             // Intercept comments
             if is_comment {
                 self.push(c); if c == ')' { break; } else { continue }; }
@@ -94,8 +118,8 @@ impl Iterator for TokenIterator {
                 (false, false) => self.push(c),      // c is a character of the token
                 (true, false) => break,              // c is trailing whitespace
             }
-            // Allow literal values to be attached to the end of the previous token
-            if self.source.len() > 0 && c == ':' { break; }
+            // Allow literal values to be chained to the end of the previous token
+            if self.source.len() > 0 && c == ':' { break }
         }
 
         // If no source characters were grabbed then we have read through the entire source file