1 files changed, 66 insertions, 0 deletions
diff --git a/src/tokens/syntactic.rs b/src/tokens/syntactic.rs
new file mode 100644
index 0000000..000d178
--- /dev/null
+++ b/src/tokens/syntactic.rs
@@ -0,0 +1,66 @@
+use crate::*;
+
+
+pub struct SyntacticToken {
+    pub source: SourceSpan,
+    pub variant: SyntacticTokenVariant,
+}
+
+pub enum SyntacticTokenVariant {
+    LabelDefinition(String),
+    MacroDefinition(String),
+    MacroDefinitionTerminator,
+
+    DecimalLiteral(usize),
+    HexadecimalLiteral(usize),
+    PackedBinaryLiteral(PackedBinaryLiteral),
+
+    Comment(String),
+    ConstantExpression(ConstantExpression),
+
+    BlockOpen,
+    BlockClose,
+    Separator,
+
+    Symbol(String),
+
+    Error(SyntacticParseError),
+}
+
+#[derive(Debug)]
+pub enum SyntacticParseError {
+    InvalidHexadecimalLiteral(String),
+    InvalidSymbolIdentifier(String),
+    UnterminatedComment,
+    UnterminatedConstantExpression,
+}
+
+
+impl std::fmt::Debug for SyntacticToken {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> {
+        use SyntacticTokenVariant::*;
+        let start = &self.source.in_merged;
+        let name = match &self.variant {
+            LabelDefinition(name) => format!("LabelDefinition({name})"),
+            MacroDefinition(name) => format!("MacroDefinition({name})"),
+            MacroDefinitionTerminator => format!("MacroDefinitionTerminator"),
+
+            DecimalLiteral(value) => format!("DecimalLiteral({value})"),
+            HexadecimalLiteral(value) => format!("HexadecimalLiteral(0x{value:x})"),
+            PackedBinaryLiteral(pbl) => format!("PackedBinaryLiteral({pbl})"),
+
+            Comment(_) => format!("Comment"),
+            ConstantExpression(expr) => format!("ConstantExpression({expr:?})"),
+
+            BlockOpen => format!("BlockOpen"),
+            BlockClose => format!("BlockClose"),
+            Separator => format!("Separator"),
+
+            Symbol(name) => format!("Symbol({name})"),
+
+            Error(error) => format!("Error({error:?})"),
+        };
+
+        write!(f, "{start} {name}")
+    }
+}