revamped lexer code to be more flexible

This commit is contained in:
Valerie Wolfe 2024-04-02 23:33:50 -04:00
parent ff05782056
commit e4ad03774e

View file

@ -2,51 +2,65 @@ using System.Text.RegularExpressions;
using Dungeoneer.Error; using Dungeoneer.Error;
namespace Dungeoneer.Lexing { namespace Dungeoneer.Interpreter {
public static class Lexer { public static class Lexer {
public static readonly char[] Delimiters = { ' ', '+', '-', '*', '/', '^' };
public static readonly char[] Operators = { '+', '-', '*', '/', '^' };
public static TokenSet Tokenize(string text) { public static TokenSet Tokenize(string text) {
var output = new TokenSet(); var output = new TokenSet();
var parts = text.Trim().Split(" ", 0x11);
foreach(string part in parts) {
var diceMatch = DiceToken.Match(part); /* - iterate over input characters:
if(diceMatch.Success) { * - create a buffer of text characters
output.Add(new DiceToken(diceMatch)); * - if a delimiter is found:
continue; * - consume the buffer into a token and add to output
* - if the delimiter was an operator, add its token to output
* - when the string terminates, consume the buffer
*/
string buffer = "";
foreach(char c in text) {
if(Delimiters.Contains(c)) {
if(buffer.Length != 0) {
Console.WriteLine(buffer);
var token = Match(buffer);
output.Add(token);
} }
var numberMatch = NumberToken.Match(part); if(Operators.Contains(c))
if(numberMatch.Success) { output.Add(new OperatorToken(c));
output.Add(new NumberToken(numberMatch)); buffer = "";
continue; } else
buffer += c;
} }
if(buffer.Length != 0)
output.Add(Match(buffer));
var dcMatch = DcToken.Match(part);
if(dcMatch.Success) {
output.Add(new DcToken(dcMatch));
continue;
}
var operatorMatch = OperatorToken.Match(part);
if(operatorMatch.Success) {
output.Add(new OperatorToken(operatorMatch));
continue;
}
var varMatch = VarToken.Match(part);
if(varMatch.Success) {
output.Add(new VarToken(varMatch));
continue;
}
throw new UnmatchedTokenException(part);
}
return output; return output;
} }
public static Token Match(string text) {
var diceMatch = DiceToken.Match(text);
if(diceMatch.Success)
return new DiceToken(diceMatch);
var numberMatch = NumberToken.Match(text);
if(numberMatch.Success)
return new NumberToken(numberMatch);
var dcMatch = DcToken.Match(text);
if(dcMatch.Success)
return new DcToken(dcMatch);
var varMatch = VarToken.Match(text);
if(varMatch.Success)
return new VarToken(varMatch);
throw new UnmatchedTokenException(text);
}
} }
public class TokenSet : List<Token> { public class TokenSet : List<Token> {
@ -57,7 +71,7 @@ namespace Dungeoneer.Lexing {
var output = ""; var output = "";
foreach(Token token in this) foreach(Token token in this)
output += $"{token} "; output += $"{token} ";
return output; return output.Trim();
} }
} }
@ -79,17 +93,11 @@ namespace Dungeoneer.Lexing {
} }
public class OperatorToken : Token { public class OperatorToken : Token {
public string dummy; public char Value;
internal static readonly Regex Pattern = new Regex(@"([\+\-\*\/])"); public OperatorToken(char op) { Value = op; }
public OperatorToken(Match match) { public override string ToString() { return Value.ToString(); }
dummy = match.Groups[1].Value;
}
public override string ToString() { return dummy; }
public static Match Match(string text) { return Pattern.Match(text); }
} }