Separate syntactic and semantic token types by namespace

This will allow type names to be shared by both types of token.

5 files changed, 50 insertions, 52 deletions

diff --git a/src/compiler.rs b/src/compiler.rs
index 068c6d5..2ad9145 100644
--- a/src/compiler.rs
+++ b/src/compiler.rs
@@ -58,7 +58,7 @@ impl Compiler {
 
 /// Parse all symbols from a source code string.
 fn parse_symbols(source_code: &str, path: Option<&Path>) -> Vec<Symbol> {
-    use SyntacticTokenVariant as SynVar;
+    use syntactic::*;
     use DefinitionType::*;
     use SymbolRole::*;
     let mut symbols = Vec::new();
@@ -82,36 +82,36 @@ fn parse_symbols(source_code: &str, path: Option<&Path>) -> Vec<Symbol> {
 
     for token in SyntacticParser::from_source_code(&source_code, path) {
         match token.variant {
-            SynVar::MacroDefinition(name) => {
+            TokenVariant::MacroDefinition(name) => {
                 push!(name.clone(), token.source, Definition(MustPrecedeReference));
                 macro_name = Some(name);
                 parse_arg_list = true;
             }
-            SynVar::MacroDefinitionTerminator => {
+            TokenVariant::MacroDefinitionTerminator => {
                 macro_name = None;
             }
-            SynVar::LabelDefinition(name) => {
+            TokenVariant::LabelDefinition(name) => {
                 push!(name.clone(), token.source, Definition(CanFollowReference));
             }
-            SynVar::Symbol(name) => if parse_arg_list && after_separator {
+            TokenVariant::Symbol(name) => if parse_arg_list && after_separator {
                 push!(name, token.source, Definition(MustPrecedeReference));
             } else {
                 parse_arg_list = false;
                 push!(name, token.source, Reference);
             }
-            SynVar::Separator => {
+            TokenVariant::Separator => {
                 after_separator = true;
                 continue;
             }
-            SynVar::BlockOpen | SynVar::BlockClose => {
+            TokenVariant::BlockOpen | TokenVariant::BlockClose => {
                 continue;
             }
-            SynVar::PackedBinaryLiteral(pbl) => {
+            TokenVariant::PackedBinaryLiteral(pbl) => {
                 for field in pbl.fields {
                     push!(field.name.to_string(), field.source, Reference)
                 }
             }
-            SynVar::ConstantExpression(expr) => {
+            TokenVariant::ConstantExpression(expr) => {
                 use ConstantExpressionTokenVariant as TokenVar;
                 for token in expr.tokens {
                     if let TokenVar::SymbolReference(name) = token.variant {
diff --git a/src/main.rs b/src/main.rs
index 0086496..32f4f5d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -129,11 +129,13 @@ fn write_bytes_and_exit<P: AsRef<Path>>(bytes: &[u8], path: Option<&P>) -> ! {
 }
 
 
-fn report_syntactic_errors(syntactic_tokens: &[SyntacticToken], source_code: &str) {
+fn report_syntactic_errors(syntactic_tokens: &[syntactic::Token], source_code: &str) {
+    use syntactic::*;
+
     for token in syntactic_tokens {
         let context = Context { source_code: &source_code, source: &token.source };
         match &token.variant {
-            SyntacticTokenVariant::ConstantExpression(expr) => for t in &expr.tokens {
+            TokenVariant::ConstantExpression(expr) => for t in &expr.tokens {
                 let context = Context { source_code: &source_code, source: &t.source };
                 if let ConstantExpressionTokenVariant::Error(err) = &t.variant {
                     let ConstantExpressionParseError::InvalidHexadecimalLiteral(hex) = err;
@@ -141,7 +143,7 @@ fn report_syntactic_errors(syntactic_tokens: &[SyntacticToken], source_code: &st
                     report_source_issue(LogLevel::Error, &context, &message);
                 }
             }
-            SyntacticTokenVariant::PackedBinaryLiteral(pbl) => for e in &pbl.errors {
+            TokenVariant::PackedBinaryLiteral(pbl) => for e in &pbl.errors {
                 let context = Context { source_code: &source_code, source: &e.source };
                 match e.variant {
                     PackedBinaryLiteralParseErrorVariant::DuplicateFieldName(name) => {
@@ -154,20 +156,20 @@ fn report_syntactic_errors(syntactic_tokens: &[SyntacticToken], source_code: &st
                     }
                 }
             }
-            SyntacticTokenVariant::Error(err) => match err {
-                SyntacticParseError::InvalidHexadecimalLiteral(hex) => {
+            TokenVariant::Error(err) => match err {
+                ParseError::InvalidHexadecimalLiteral(hex) => {
                     let message = format!("Invalid hexadecimal literal {hex:?}");
                     report_source_issue(LogLevel::Error, &context, &message);
                 }
-                SyntacticParseError::InvalidSymbolIdentifier(name) => {
+                ParseError::InvalidSymbolIdentifier(name) => {
                     let message = format!("Invalid identifier {name:?}");
                     report_source_issue(LogLevel::Error, &context, &message);
                 }
-                SyntacticParseError::UnterminatedComment => {
+                ParseError::UnterminatedComment => {
                     let message = format!("Unterminated comment");
                     report_source_issue(LogLevel::Error, &context, &message);
                 }
-                SyntacticParseError::UnterminatedConstantExpression => {
+                ParseError::UnterminatedConstantExpression => {
                     let message = format!("Unterminated constant expression");
                     report_source_issue(LogLevel::Error, &context, &message);
                 }
diff --git a/src/parsers/syntactic.rs b/src/parsers/syntactic.rs
index 443e47e..c98a592 100644
--- a/src/parsers/syntactic.rs
+++ b/src/parsers/syntactic.rs
@@ -1,4 +1,5 @@
 use crate::*;
+use syntactic::*;
 
 
 pub struct SyntacticParser {
@@ -23,76 +24,73 @@ impl SyntacticParser {
 
 
 impl Iterator for SyntacticParser {
-    type Item = SyntacticToken;
+    type Item = Token;
 
     /// Sequentially parse tokens from the source code.
-    fn next(&mut self) -> Option<SyntacticToken> {
-        use SyntacticTokenVariant as SynVar;
-        use SyntacticParseError as SynErr;
+    fn next(&mut self) -> Option<Token> {
         let t = &mut self.tokeniser;
-
         t.drop_whitespace();
         t.mark_start_position();
 
         let variant = match t.eat_char()? {
             '@' => {
                 self.label_name = t.eat_token();
-                SynVar::LabelDefinition(self.label_name.clone())
+                TokenVariant::LabelDefinition(self.label_name.clone())
             }
             '&' => {
                 let token = t.eat_token();
-                SynVar::LabelDefinition(format!("{}/{token}", self.label_name))
+                TokenVariant::LabelDefinition(format!("{}/{token}", self.label_name))
             }
             '%' => {
                 let macro_name = t.eat_token();
                 self.macro_name = Some(macro_name.clone());
-                SynVar::MacroDefinition(macro_name)
+                TokenVariant::MacroDefinition(macro_name)
             }
             ';' => {
                 self.macro_name = None;
-                SynVar::MacroDefinitionTerminator
+                TokenVariant::MacroDefinitionTerminator
             }
             '[' => match t.eat_to_delimiter(']') {
                 Some(string) => {
                     let constant = ConstantExpression::from_str(&string, t);
-                    SynVar::ConstantExpression(constant)
+                    TokenVariant::ConstantExpression(constant)
                 }
-                None => SynVar::Error(SynErr::UnterminatedConstantExpression),
+                None => TokenVariant::Error(ParseError::UnterminatedConstantExpression),
             }
-            '{' => SynVar::BlockOpen,
-            '}' => SynVar::BlockClose,
+            '{' => TokenVariant::BlockOpen,
+            '}' => TokenVariant::BlockClose,
             '(' => match t.eat_to_delimiter(')') {
-                Some(string) => SynVar::Comment(string),
-                None => SynVar::Error(SynErr::UnterminatedComment),
+                Some(string) => TokenVariant::Comment(string),
+                None => TokenVariant::Error(ParseError::UnterminatedComment),
             }
             '#' => {
                 let token = t.eat_token();
                 let pbl = PackedBinaryLiteral::from_str(&token, t);
-                SynVar::PackedBinaryLiteral(pbl)
+                TokenVariant::PackedBinaryLiteral(pbl)
             },
             '~' => {
                 let token = t.eat_token();
-                SynVar::Symbol(format!("{}/{token}", self.label_name))
+                TokenVariant::Symbol(format!("{}/{token}", self.label_name))
             }
-            ':' => SynVar::Separator,
+            ':' => TokenVariant::Separator,
             c => {
                 let token = format!("{c}{}", t.eat_token());
                 if let Some(hex_string) = token.strip_prefix("0x") {
                     match usize::from_str_radix(hex_string, 16) {
-                        Ok(hex) => SynVar::HexadecimalLiteral(hex),
-                        Err(_) => SynVar::Error(SynErr::InvalidHexadecimalLiteral(token)),
+                        Ok(hex) => TokenVariant::HexadecimalLiteral(hex),
+                        Err(_) => TokenVariant::Error(ParseError::InvalidHexadecimalLiteral(token)),
                     }
                 } else {
                     match usize::from_str_radix(&token, 10) {
-                        Ok(value) => SynVar::DecimalLiteral(value),
-                        Err(_) => SynVar::Symbol(token),
+                        Ok(value) => TokenVariant::DecimalLiteral(value),
+                        Err(_) => TokenVariant::Symbol(token),
                     }
                 }
             }
         };
 
         // Parse source path comments.
-        if let SynVar::Comment(comment) = &variant {
+        if let TokenVariant::Comment(comment) = &variant {
             // Check if the comment fills the entire line.
             if t.start_position.column == 0 && t.end_of_line() {
                 if let Some(path) = comment.strip_prefix(": ") {
@@ -103,6 +101,6 @@ impl Iterator for SyntacticParser {
         }
 
         let source = t.mark_end_position();
-        Some( SyntacticToken { source, variant } )
+        Some( Token { source, variant } )
     }
 }
diff --git a/src/tokens/mod.rs b/src/tokens/mod.rs
index 65f361c..edb7c19 100644
--- a/src/tokens/mod.rs
+++ b/src/tokens/mod.rs
@@ -1,8 +1,6 @@
-mod syntactic;
-pub use syntactic::*;
+pub mod syntactic;
 
-mod semantic;
-pub use semantic::*;
+pub mod semantic;
 
 mod constant_expression;
 pub use constant_expression::*;
diff --git a/src/tokens/syntactic.rs b/src/tokens/syntactic.rs
index 000d178..162f1c0 100644
--- a/src/tokens/syntactic.rs
+++ b/src/tokens/syntactic.rs
@@ -1,12 +1,12 @@
 use crate::*;
 
 
-pub struct SyntacticToken {
+pub struct Token {
     pub source: SourceSpan,
-    pub variant: SyntacticTokenVariant,
+    pub variant: TokenVariant,
 }
 
-pub enum SyntacticTokenVariant {
+pub enum TokenVariant {
     LabelDefinition(String),
     MacroDefinition(String),
     MacroDefinitionTerminator,
@@ -24,11 +24,11 @@ pub enum SyntacticTokenVariant {
 
     Symbol(String),
 
-    Error(SyntacticParseError),
+    Error(ParseError),
 }
 
 #[derive(Debug)]
-pub enum SyntacticParseError {
+pub enum ParseError {
     InvalidHexadecimalLiteral(String),
     InvalidSymbolIdentifier(String),
     UnterminatedComment,
@@ -36,9 +36,9 @@ pub enum SyntacticParseError {
 }
 
 
-impl std::fmt::Debug for SyntacticToken {
+impl std::fmt::Debug for Token {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
-        use SyntacticTokenVariant::*;
+        use TokenVariant::*;
         let start = &self.source.in_merged;
         let name = match &self.variant {
             LabelDefinition(name) => format!("LabelDefinition({name})"),