From 53dffaf3fe18f70151b23418ff80ae8904fae779 Mon Sep 17 00:00:00 2001 From: Ben Bridle Date: Sat, 8 Mar 2025 16:48:33 +1300 Subject: Make Tokeniser::eat_until even more general The eat_until method of Tokeniser has been renamed to track_until, and it no longer consumes characters by itself. The predicate closure is no longer limited to checking one character at a time, it can look ahead for multi-character sequences when used with the new eat_if method. --- src/tokeniser.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/tokeniser.rs b/src/tokeniser.rs index 3608e9e..e09a439 100644 --- a/src/tokeniser.rs +++ b/src/tokeniser.rs @@ -139,20 +139,17 @@ impl Tokeniser { return token; } - /// Consume and return all characters up to and including the delimiter. + /// Return all characters found until the predicate returns true. /// Returns None if end of source is reached before delimiter is found. - pub fn eat_until(&mut self, mut predicate: impl FnMut(&char) -> bool) -> Option { - let mut token = String::new(); - while let Some(c) = self.eat_char() { - match predicate(&c) { - true => { - self.end = self.prev; - return Some(token); - } - false => token.push(c), - } + pub fn track_until(&mut self, mut predicate: impl FnMut(&mut Self) -> bool) -> Option { + let start = self.mark; + let mut end = self.mark; + while !predicate(self) { + self.peek_char()?; + end = self.mark; } - return None; + self.end = self.prev; + return Some(self.get_chars(&start, &end).iter().collect()); } /// Returns true if the remainder of the line is whitespace. -- cgit v1.2.3-70-g09d2