use crate::*;
use std::path::PathBuf;
/// Break a character stream down into individual tokens.
pub struct Tokeniser {
/// Characters waiting to be parsed, in reverse order.
pub chars: Vec<char>,
/// Path of the whole source file.
pub source_path: Option<PathBuf>,
/// Original path of the embedded source file.
pub embedded_path: Option<PathBuf>,
/// Line where the embedded source file begins.
pub embedded_first_line: usize,
/// Position of the next character to be consumed.
pub position: SourcePosition,
/// Position of the most recently consumed character.
pub prev_position: SourcePosition,
/// Position of the first character of the current token.
pub start_position: SourcePosition,
/// The source characters consumed for the current token.
pub consumed: String,
/// List of characters that start a new token.
pub delimiters: Vec<char>,
/// List of characters that terminate a token.
pub terminators: Vec<char>,
}
impl Tokeniser {
pub fn new<P: Into<PathBuf>>(source_code: &str, path: Option<P>) -> Self {
Self {
chars: source_code.chars().rev().collect(),
source_path: path.map(|p| p.into()),
embedded_path: None,
embedded_first_line: 0,
position: SourcePosition::ZERO,
prev_position: SourcePosition::ZERO,
start_position: SourcePosition::ZERO,
consumed: String::new(),
delimiters: Vec::new(),
terminators: Vec::new(),
}
}
pub fn add_delimiters(&mut self, delimiters: &[char]) {
self.delimiters.extend_from_slice(delimiters);
}
pub fn add_terminators(&mut self, terminators: &[char]) {
self.terminators.extend_from_slice(terminators);
}
/// Return the next character without consuming it.
pub fn peek_char(&self) -> Option<char> {
self.chars.last().copied()
}
/// Consume and return the next character.
pub fn eat_char(&mut self) -> Option<char> {
let option = self.chars.pop();
if let Some(c) = option {
self.prev_position = self.position;
self.position.advance(c);
self.consumed.push(c);
}
return option;
}
/// Remove the next character.
pub fn drop_char(&mut self) {
if let Some(c) = self.chars.pop() {
self.prev_position = self.position;
self.position.advance(c);
}
}
/// Remove whitespace.
pub fn drop_whitespace(&mut self) {
while let Some(c) = self.peek_char() {
match c.is_whitespace() {
true => self.drop_char(),
false => break,
}
}
}
/// Remove a full token from the queue.
pub fn eat_token(&mut self) -> String {
let mut token = String::new();
while let Some(peek) = self.peek_char() {
if peek.is_whitespace() || self.delimiters.contains(&peek) {
break;
}
let c = self.eat_char().unwrap();
token.push(c);
if self.terminators.contains(&c) {
break;
}
}
return token;
}
/// Consume and return all characters up to and including the delimiter.
/// Returns None if end of source is reached before delimiter is found.
pub fn eat_to_delimiter(&mut self, delim: char) -> Option<String> {
let mut token = String::new();
while let Some(c) = self.eat_char() {
self.consumed.push(c);
match c == delim {
true => return Some(token),
false => token.push(c),
}
}
return None;
}
/// Returns true if the remainder of the line is whitespace.
pub fn end_of_line(&self) -> bool {
for c in self.chars.iter().rev() {
if *c == '\n' {
return true;
}
if !c.is_whitespace() {
return false
}
}
return true;
}
/// Mark the next character to be consumed as the start character.
pub fn mark_start_position(&mut self) {
self.start_position = self.position;
self.consumed.clear();
}
/// Mark the previously-consumed character as the start character.
pub fn mark_prev_start_position(&mut self) {
self.start_position = self.prev_position;
let c = self.consumed.chars().last();
self.consumed.clear();
// Keep the previously pushed character.
if let Some(c) = c {
self.consumed.push(c);
}
}
/// Only call this once per span, it consumes the token string.
pub fn get_source_span(&mut self) -> SourceSpan {
let in_merged = SourceLocation {
path: self.source_path.to_owned(),
start: self.start_position,
end: self.prev_position,
};
let in_source = if self.start_position.line >= self.embedded_first_line {
if let Some(embedded_path) = &self.embedded_path {
let offset = self.embedded_first_line;
Some(
SourceLocation {
path: Some(embedded_path.to_owned()),
start: SourcePosition {
line: in_merged.start.line.saturating_sub(offset),
column: in_merged.start.column,
},
end: SourcePosition {
line: in_merged.end.line.saturating_sub(offset),
column: in_merged.end.column,
}
}
)
} else {
None
}
} else {
None
};
let string = std::mem::take(&mut self.consumed);
SourceSpan { string, in_merged, in_source }
}
}