From 2d67476d48fcb33cd1c59cbc2e0f82872bc0c217 Mon Sep 17 00:00:00 2001 From: Ben Bridle Date: Mon, 10 Feb 2025 11:54:32 +1300 Subject: Add tokeniser method to mark previous char as start of token This enables a parsing technique where characters can be marked as the start of a token after they've been eaten, instead of having to determine ahead of time via defensively peeking. --- src/tokeniser.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src') diff --git a/src/tokeniser.rs b/src/tokeniser.rs index 7d8ddfd..6ae9055 100644 --- a/src/tokeniser.rs +++ b/src/tokeniser.rs @@ -129,8 +129,21 @@ impl Tokeniser { return true; } + /// Mark the next character to be consumed as the start character. pub fn mark_start_position(&mut self) { self.start_position = self.position; + self.consumed.clear(); + } + + /// Mark the previously-consumed character as the start character. + pub fn mark_prev_start_position(&mut self) { + self.start_position = self.prev_position; + let c = self.consumed.chars().last(); + self.consumed.clear(); + // Keep the previously pushed character. + if let Some(c) = c { + self.consumed.push(c); + } } /// Only call this once per span, it consumes the token string. -- cgit v1.2.3-70-g09d2