Implement lists as a first-class type

Previously, strings were implemented as lists of integers, but there
was no way to create an arbitrary list of integers to pass to a macro
invocation.

This commit extends the []-delimited expression syntax, treating
expressions that contain only integers and invocations as lists of
integers. Strings are implemented as a sub-type of list.

3 files changed, 104 insertions, 40 deletions

diff --git a/src/stages/compiler.rs b/src/stages/compiler.rs
index d421e83..7dfebd0 100644
--- a/src/stages/compiler.rs
+++ b/src/stages/compiler.rs
@@ -94,8 +94,8 @@ impl SymbolParser {
                         MacroDefinitionBody::Integer(integer) => {
                             self.parse_integer_token(&integer, &integer.source)
                         }
-                        MacroDefinitionBody::String(string) => {
-                            self.parse_string_token(&string, &string.source)
+                        MacroDefinitionBody::List(list) => {
+                            self.parse_list_token(&list, &list.source)
                         }
                         MacroDefinitionBody::Invocation(invocation) => {
                             self.parse_invocation(&invocation, &invocation.source)
@@ -123,6 +123,9 @@ impl SymbolParser {
                 ExpressionToken::IntegerToken(integer) => {
                     self.parse_integer_token(integer, source);
                 }
+                ExpressionToken::ListToken(list) => {
+                    self.parse_list_token(list, source);
+                }
                 ExpressionToken::Invocation(invocation) => {
                     self.parse_invocation(invocation, source);
                 }
@@ -148,8 +151,8 @@ impl SymbolParser {
                 InvocationArgument::BlockToken(block) => {
                     self.parse_block_token(block, &source);
                 }
-                InvocationArgument::StringToken(string) => {
-                    self.parse_string_token(string, &source);
+                InvocationArgument::ListToken(list) => {
+                    self.parse_list_token(list, &source);
                 },
                 InvocationArgument::Invocation(invocation) => {
                     self.parse_invocation(invocation, &source);
@@ -208,12 +211,17 @@ impl SymbolParser {
         }
     }
 
-    fn parse_string_token(&mut self, token: &StringToken, source: &SourceSpan) {
+    fn parse_list_token(&mut self, token: &ListToken, source: &SourceSpan) {
         match &token {
-            StringToken::Invocation(invocation) => {
+            ListToken::Invocation(invocation) => {
                 self.parse_invocation(&invocation, source)
             }
-            StringToken::StringLiteral(_) => (),
+            ListToken::ListLiteral(integers) => {
+                for integer in integers {
+                    self.parse_integer_token(&integer, source)
+                }
+            }
+            ListToken::StringLiteral(_) => (),
         }
     }
 }
diff --git a/src/stages/semantic.rs b/src/stages/semantic.rs
index b72cfa8..fdd1766 100644
--- a/src/stages/semantic.rs
+++ b/src/stages/semantic.rs
@@ -144,9 +144,9 @@ impl SemanticParser {
                     MacroDefinitionBody::Block(mut tokens) => {
                         block_tokens.append(&mut tokens);
                     }
-                    MacroDefinitionBody::String(string) => {
+                    MacroDefinitionBody::List(list) => {
                         let error = SemanticError::ExpectedBlock(location);
-                        let tracked = Tracked::from(error, string.source);
+                        let tracked = Tracked::from(error, list.source);
                         self.errors.push(tracked);
                     }
                     MacroDefinitionBody::Invocation(invocation) => {
@@ -184,9 +184,9 @@ impl SemanticParser {
                 Some(MacroDefinitionBody::Integer(tracked))
             }
             SyntacticToken::StringLiteral(value) => {
-                let token = StringToken::StringLiteral(value);
+                let token = ListToken::StringLiteral(value);
                 let tracked = Tracked::from(token, source);
-                Some(MacroDefinitionBody::String(tracked))
+                Some(MacroDefinitionBody::List(tracked))
             }
             SyntacticToken::WordTemplate(word_template) => {
                 let token = BlockToken::WordTemplate(word_template);
@@ -205,9 +205,16 @@ impl SemanticParser {
                 let mut parser = SemanticParser::from(tokens, self.namespace.clone());
                 let expression = parser.parse_expression();
                 self.pull_from(parser);
-                let token = IntegerToken::Expression(expression);
-                let tracked = Tracked::from(token, source);
-                Some(MacroDefinitionBody::Integer(tracked))
+                if expression.is_list() {
+                    let list = expression.to_list();
+                    let token = ListToken::ListLiteral(list);
+                    let tracked = Tracked::from(token, source);
+                    Some(MacroDefinitionBody::List(tracked))
+                } else {
+                    let token = IntegerToken::Expression(expression);
+                    let tracked = Tracked::from(token, source);
+                    Some(MacroDefinitionBody::Integer(tracked))
+                }
             }
             SyntacticToken::Symbol(symbol) => {
                 let name = self.resolve_symbol_name(symbol, &source)?;
@@ -265,9 +272,9 @@ impl SemanticParser {
                 self.errors.push(Tracked::from(error, token.source));
                 None
             }
-            MacroDefinitionBody::String(string) => {
+            MacroDefinitionBody::List(list) => {
                 let error = SemanticError::ExpectedInteger(location);
-                self.errors.push(Tracked::from(error, string.source));
+                self.errors.push(Tracked::from(error, list.source));
                 None
             }
         }
@@ -291,9 +298,9 @@ impl SemanticParser {
                 self.errors.push(Tracked::from(error, integer.source));
                 None
             }
-            MacroDefinitionBody::String(string) => {
+            MacroDefinitionBody::List(list) => {
                 let error = SemanticError::ExpectedBlock(location);
-                self.errors.push(Tracked::from(error, string.source));
+                self.errors.push(Tracked::from(error, list.source));
                 None
             }
         }
@@ -372,8 +379,8 @@ impl SemanticParser {
         let source = token.source;
         match token.value {
             SyntacticToken::StringLiteral(string_literal) => {
-                let string = StringToken::StringLiteral(string_literal);
-                let argument = InvocationArgument::StringToken(string);
+                let string = ListToken::StringLiteral(string_literal);
+                let argument = InvocationArgument::ListToken(string);
                 Some(Tracked::from(argument, source))
             }
             SyntacticToken::IntegerLiteral(value) => {
@@ -385,9 +392,16 @@ impl SemanticParser {
                 let mut parser = SemanticParser::from(tokens, self.namespace.clone());
                 let expression = parser.parse_expression();
                 self.pull_from(parser);
-                let integer = IntegerToken::Expression(expression);
-                let argument = InvocationArgument::IntegerToken(integer);
-                Some(Tracked::from(argument, source))
+                if expression.is_list() {
+                    let list = expression.to_list();
+                    let token = ListToken::ListLiteral(list);
+                    let argument = InvocationArgument::ListToken(token);
+                    Some(Tracked::from(argument, source))
+                } else {
+                    let integer = IntegerToken::Expression(expression);
+                    let argument = InvocationArgument::IntegerToken(integer);
+                    Some(Tracked::from(argument, source))
+                }
             }
             SyntacticToken::BlockLiteral(tokens) => {
                 let mut parser = SemanticParser::from(tokens, self.namespace.clone());
diff --git a/src/stages/semantic_tokens.rs b/src/stages/semantic_tokens.rs
index 95a7f21..a873df0 100644
--- a/src/stages/semantic_tokens.rs
+++ b/src/stages/semantic_tokens.rs
@@ -37,7 +37,7 @@ impl std::fmt::Display for ArgumentType {
 pub enum MacroDefinitionBody {
     Integer(Tracked<IntegerToken>),
     Block(Vec<Tracked<BlockToken>>),
-    String(Tracked<StringToken>),
+    List(Tracked<ListToken>),
     Invocation(Tracked<Invocation>),
 }
 
@@ -58,6 +58,7 @@ pub struct Expression {
 
 pub enum ExpressionToken {
     IntegerToken(Box<IntegerToken>),
+    ListToken(ListToken),
     Invocation(Invocation),
     Operator(Operator),
 }
@@ -71,8 +72,9 @@ pub enum BlockToken {
     Invocation(Invocation),
 }
 
-pub enum StringToken {
+pub enum ListToken {
     StringLiteral(StringLiteral),
+    ListLiteral(Vec<Tracked<IntegerToken>>),
     Invocation(Invocation),
 }
 
@@ -84,13 +86,48 @@ pub struct Invocation {
 pub enum InvocationArgument {
     IntegerToken(IntegerToken),
     BlockToken(BlockToken),
-    StringToken(StringToken),
+    ListToken(ListToken),
     Invocation(Invocation),
 }
 
+
+impl Expression {
+    pub fn is_list(&self) -> bool {
+        self.tokens.iter().all(|t| {
+            match t.value {
+                ExpressionToken::IntegerToken(_) => true,
+                ExpressionToken::Invocation(_) => true,
+                ExpressionToken::ListToken(_) => false,
+                ExpressionToken::Operator(_) => false,
+            }
+        })
+    }
+
+    pub fn to_list(self) -> Vec<Tracked<IntegerToken>> {
+        let mut list = Vec::new();
+        for token in self.tokens {
+            let source = token.source;
+            match token.value {
+                ExpressionToken::IntegerToken(token) => {
+                    let tracked = Tracked::from(*token, source);
+                    list.push(tracked);
+                }
+                ExpressionToken::Invocation(invocation) => {
+                    let token = IntegerToken::Invocation(invocation);
+                    list.push(Tracked::from(token, source));
+                }
+                ExpressionToken::ListToken(_) => unreachable!(
+                    "Could not convert expression containing a list token to a list"),
+                ExpressionToken::Operator(_) => unreachable!(
+                    "Could not convert expression containing an operator to a list"),
+            };
+        }
+        return list;
+    }
+}
+
+
 pub enum SemanticError {
-    MisplacedStringLiteral,
-    MisplacedListLiteral,
     MisplacedSeparator,
     MisplacedMacroDefinition,
 
@@ -149,10 +186,6 @@ pub fn report_semantic_errors(errors: &[Tracked<SemanticError>], source_code: &s
 fn report_semantic_error(error: &Tracked<SemanticError>, source_code: &str) {
     let context = Context { source_code: &source_code, source: &error.source };
     let message = match &error.value {
-        SemanticError::MisplacedStringLiteral =>
-            "String literals can only be used as invocation arguments",
-        SemanticError::MisplacedListLiteral =>
-            "List literals can only be used as invocation arguments",
         SemanticError::MisplacedSeparator =>
             "Separators can only be used for constructing an argument list",
         SemanticError::MisplacedMacroDefinition =>
@@ -196,8 +229,8 @@ pub fn print_semantic_token(i: usize, token: &SemanticToken) {
                 MacroDefinitionBody::Block(tokens) => {
                     print_block(i+1, tokens);
                 }
-                MacroDefinitionBody::String(string) => {
-                    print_string_token(i+1, string);
+                MacroDefinitionBody::List(list) => {
+                    print_list_token(i+1, list);
                 }
                 MacroDefinitionBody::Invocation(invocation) => {
                     print_invocation(i+1, invocation);
@@ -266,8 +299,8 @@ fn print_invocation(i: usize, invocation: &Invocation) {
 
 fn print_invocation_argument(i: usize, argument: &InvocationArgument) {
     match &argument {
-        InvocationArgument::StringToken(string) => {
-            print_string_token(i, string)
+        InvocationArgument::ListToken(list) => {
+            print_list_token(i, list)
         }
         InvocationArgument::IntegerToken(integer) => {
             print_integer_token(i, integer)
@@ -295,12 +328,18 @@ fn print_integer_token(i: usize, integer: &IntegerToken) {
     }
 }
 
-fn print_string_token(i: usize, string: &StringToken) {
+fn print_list_token(i: usize, string: &ListToken) {
     match string {
-        StringToken::StringLiteral(string_literal) => {
-            indent!(i, "String({string_literal})")
+        ListToken::StringLiteral(string_literal) => {
+            indent!(i, "StringLiteral({string_literal})")
         }
-        StringToken::Invocation(invocation) => {
+        ListToken::ListLiteral(integers) => {
+            indent!(i, "ListLiteral");
+            for integer in integers {
+                print_integer_token(i+1, integer);
+            }
+        }
+        ListToken::Invocation(invocation) => {
             print_invocation(i, invocation)
         }
     }
@@ -313,6 +352,9 @@ fn print_expression(i: usize, expression: &Expression) {
             ExpressionToken::IntegerToken(integer) => {
                 print_integer_token(i+1, &integer)
             }
+            ExpressionToken::ListToken(list) => {
+                print_list_token(i+1, &list)
+            }
             ExpressionToken::Invocation(invocation) => {
                 print_invocation(i+1, &invocation);
             }