From 2d67476d48fcb33cd1c59cbc2e0f82872bc0c217 Mon Sep 17 00:00:00 2001
From: Ben Bridle <bridle.benjamin@gmail.com>
Date: Mon, 10 Feb 2025 11:54:32 +1300
Subject: Add tokeniser method to mark previous char as start of token
This enables a parsing technique where characters can be marked as
the start of a token after they've been eaten, instead of having to
determine ahead of time via defensively peeking.
---
src/tokeniser.rs | 13 +++++++++++++
1 file changed, 13 insertions(+)
(limited to 'src/tokeniser.rs')
diff --git a/src/tokeniser.rs b/src/tokeniser.rs
index 7d8ddfd..6ae9055 100644
--- a/src/tokeniser.rs
+++ b/src/tokeniser.rs
@@ -129,8 +129,21 @@ impl Tokeniser {
return true;
}
+ /// Mark the next character to be consumed as the start character.
pub fn mark_start_position(&mut self) {
self.start_position = self.position;
+ self.consumed.clear();
+ }
+
+ /// Mark the previously-consumed character as the start character.
+ pub fn mark_prev_start_position(&mut self) {
+ self.start_position = self.prev_position;
+ let c = self.consumed.chars().last();
+ self.consumed.clear();
+ // Keep the previously pushed character.
+ if let Some(c) = c {
+ self.consumed.push(c);
+ }
}
/// Only call this once per span, it consumes the token string.
--
cgit v1.2.3-70-g09d2