diff options
Diffstat (limited to 'src/tokenizer.rs')
| -rw-r--r-- | src/tokenizer.rs | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 1fdb994..03e09e3 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -106,11 +106,12 @@ impl Iterator for TokenIterator { } // Intercept comments if is_comment { - self.push(c); if c == ')' { break; } else { continue }; } + self.push(c); if c == ')' { break } else { continue }; } else if self.skip_whitespace && c == '(' { - is_comment = true; self.mark_start(c); continue; } - // Allow a semicolon on the end of a token to be handled as a separate token - if self.source.len() > 0 && c == ';' { break; } + is_comment = true; self.mark_start(c); continue } + + // Allow a semicolon at the end of a token to be handled as a separate token + if self.source.len() > 0 && c == ';' { break } // Handle the current character match (is_whitespace(c), self.skip_whitespace) { (true, true) => self.next(c), // c is the expected leading whitespace |