Add tokeniser method to mark previous char as start of token

This enables a parsing technique where characters can be marked as
the start of a token after they've been eaten, instead of having to
determine ahead of time via defensively peeking.

1 files changed, 13 insertions, 0 deletions

diff --git a/src/tokeniser.rs b/src/tokeniser.rs
index 7d8ddfd..6ae9055 100644
--- a/src/tokeniser.rs
+++ b/src/tokeniser.rs
@@ -129,8 +129,21 @@ impl Tokeniser {
         return true;
     }
 
+    /// Mark the next character to be consumed as the start character.
     pub fn mark_start_position(&mut self) {
         self.start_position = self.position;
+        self.consumed.clear();
+    }
+
+    /// Mark the previously-consumed character as the start character.
+    pub fn mark_prev_start_position(&mut self) {
+        self.start_position = self.prev_position;
+        let c = self.consumed.chars().last();
+        self.consumed.clear();
+        // Keep the previously pushed character.
+        if let Some(c) = c {
+            self.consumed.push(c);
+        }
     }
 
     /// Only call this once per span, it consumes the token string.