using System;
using System.Collections.Generic;
using System.Text;
namespace Conscript.Compiler
{
internal class ScriptLexer
{
#region Private Enumerated Types
private enum LexState
{
Space,
CommentOrDivideOrAssignDivide,
LineComment,
BlockCommentStart,
BlockCommentEnd,
AssignOrEqual,
PlusOrIncrementOrAssignPlus,
MinusOrDecrementOrAssignMinus,
MultiplyOrAssignMultiply,
PowerOrAssignPower,
ModuloOrAssignModulo,
And,
Or,
NotOrNotEqual,
GreaterOrGreaterEqual,
LessOrLessEqual,
IdentifierOrKeyword,
String,
StringEscape,
IntegerOrFloat,
Float
}
#endregion
#region Private Variables
private List<String> m_listSourceLines;
private int m_iSourceLine;
private int m_iSourceChar;
private LexState m_lexState;
#endregion
#region Private Methods
private void ThrowInvalidCharacterException(char ch)
{
throw new LexerException("Unexpected character '" + ch + "'.",
m_iSourceLine, m_iSourceChar, m_listSourceLines[m_iSourceLine]);
}
private bool EndOfSource
{
get { return m_iSourceLine >= m_listSourceLines.Count; }
}
private char ReadChar()
{
if (EndOfSource)
throw new LexerException("End of source reached.");
char ch = m_listSourceLines[m_iSourceLine][m_iSourceChar++];
if (m_iSourceChar >= m_listSourceLines[m_iSourceLine].Length)
{
m_iSourceChar = 0;
++m_iSourceLine;
}
return ch;
}
private void UndoChar()
{
if (m_iSourceLine == 0 && m_iSourceChar == 0)
throw new LexerException(
"Cannot undo char beyond start of source.");
--m_iSourceChar;
if (m_iSourceChar < 0)
{
--m_iSourceLine;
m_iSourceChar = m_listSourceLines[m_iSourceLine].Length - 1;
}
}
#endregion
#region Public Methods
public ScriptLexer(List<String> listSourceLines)
{
m_listSourceLines = new List<string>();
foreach (String strSourceLine in listSourceLines)
m_listSourceLines.Add(strSourceLine + "\r\n");
m_iSourceLine = 0;
m_iSourceChar = 0;
m_lexState = LexState.Space;
}
public List<Token> GetTokens()
{
m_iSourceLine = 0;
m_iSourceChar = 0;
m_lexState = LexState.Space;
String strToken = null;
List<Token> listTokens = new List<Token>();
while (!EndOfSource)
{
String strSourceLine = m_listSourceLines[m_iSourceLine];
char ch = ReadChar();
switch (m_lexState)
{
case LexState.Space:
switch (ch)
{
case ' ':
case '\t':
case '\r':
case '\n':
break; // ignore whitespace
case '{':
listTokens.Add(new Token(TokenType.LeftBrace,
"{", m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case '}':
listTokens.Add(new Token(TokenType.RightBrace, "}",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case '(':
listTokens.Add(new Token(TokenType.LeftPar, "(",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case ')':
listTokens.Add(new Token(TokenType.RightPar, ")",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case '[':
listTokens.Add(new Token(TokenType.LeftBracket, "[",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case ']':
listTokens.Add(new Token(TokenType.RightBracket, "]",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case '.':
listTokens.Add(new Token(TokenType.Period, ".",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case ',':
listTokens.Add(new Token(TokenType.Comma, ",",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case ';':
listTokens.Add(new Token(TokenType.SemiColon, ";",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
case '=':
m_lexState = LexState.AssignOrEqual;
break;
case '+':
m_lexState = LexState.PlusOrIncrementOrAssignPlus;
break;
case '-':
m_lexState = LexState.MinusOrDecrementOrAssignMinus;
break;
case '*':
m_lexState = LexState.MultiplyOrAssignMultiply;
break;
case '/':
m_lexState = LexState.CommentOrDivideOrAssignDivide;
break;
case '^':
m_lexState = LexState.PowerOrAssignPower;
break;
case '%':
m_lexState = LexState.ModuloOrAssignModulo;
break;
case '&':
m_lexState = LexState.And;
break;
case '|':
m_lexState = LexState.Or;
break;
case '!':
m_lexState = LexState.NotOrNotEqual;
break;
case '>':
m_lexState = LexState.GreaterOrGreaterEqual;
break;
case '<':
m_lexState = LexState.LessOrLessEqual;
break;
case '\"':
strToken = "";
m_lexState = LexState.String;
break;
case ':':
listTokens.Add(new Token(TokenType.Colon, ":",
m_iSourceLine, m_iSourceChar, strSourceLine));
break;
default:
if (ch == '_' || char.IsLetter(ch))
{
m_lexState = LexState.IdentifierOrKeyword;
strToken = "" + ch;
}
else if (char.IsDigit(ch))
{
strToken = "" + ch;
m_lexState = LexState.IntegerOrFloat;
}
else
ThrowInvalidCharacterException(ch);
break;
}
break;
case LexState.CommentOrDivideOrAssignDivide:
switch (ch)
{
case '/':
m_lexState = LexState.LineComment;
break;
case '*':
m_lexState = LexState.BlockCommentStart;
break;
case '=':
listTokens.Add(new Token(TokenType.AssignDivide, "/=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
break;
default:
listTokens.Add(new Token(TokenType.Divide, "/",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
break;
}
break;
case LexState.LineComment:
if (ch == '\n')
m_lexState = LexState.Space;
break;
case LexState.BlockCommentStart:
if (ch == '*')
m_lexState = LexState.BlockCommentEnd;
break;
case LexState.BlockCommentEnd:
if (ch == '/')
m_lexState = LexState.Space;
else
m_lexState = LexState.BlockCommentStart;
break;
case LexState.AssignOrEqual:
if (ch == '=')
{
listTokens.Add(new Token(TokenType.Equal, "==",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Assign, "=",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.PlusOrIncrementOrAssignPlus:
if (ch == '+')
{
listTokens.Add(new Token(TokenType.Increment, "++",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else if (ch == '=')
{
listTokens.Add(new Token(TokenType.AssignPlus, "+=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Plus, "+",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.MinusOrDecrementOrAssignMinus:
if (ch == '-')
{
listTokens.Add(new Token(TokenType.Decrement, "--",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else if (ch == '=')
{
listTokens.Add(new Token(TokenType.AssignMinus, "-=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Minus, "-",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.MultiplyOrAssignMultiply:
if (ch == '=')
{
listTokens.Add(new Token(TokenType.AssignMultiply, "*=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Multiply, "*",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.PowerOrAssignPower:
if (ch == '=')
{
listTokens.Add(new Token(TokenType.AssignPower, "^=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Power, "^",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.ModuloOrAssignModulo:
if (ch == '=')
{
listTokens.Add(new Token(TokenType.AssignModulo, "%=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Modulo, "%",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.And:
if (ch == '&')
{
listTokens.Add(new Token(TokenType.And, "&&",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
ThrowInvalidCharacterException(ch);
break;
case LexState.Or:
if (ch == '|')
{
listTokens.Add(new Token(TokenType.Or, "||",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
ThrowInvalidCharacterException(ch);
break;
case LexState.NotOrNotEqual:
if (ch == '=')
{
listTokens.Add(new Token(TokenType.NotEqual, "!=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Not, "!",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.GreaterOrGreaterEqual:
if (ch == '=')
{
listTokens.Add(new Token(TokenType.GreaterOrEqual, ">=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Greater, ">",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.LessOrLessEqual:
if (ch == '=')
{
listTokens.Add(new Token(TokenType.LessOrEqual, "<=",
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else
{
listTokens.Add(new Token(TokenType.Less, "<",
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.IdentifierOrKeyword:
if (ch == '_' || char.IsLetterOrDigit(ch))
strToken += ch;
else
{
TokenType tokenType;
if (strToken == "include")
tokenType = TokenType.Include;
else if (strToken == "global")
tokenType = TokenType.Global;
else if (strToken == "var")
tokenType = TokenType.Var;
else if (strToken == "yield")
tokenType = TokenType.Yield;
else if (strToken == "wait")
tokenType = TokenType.Wait;
else if (strToken == "notify")
tokenType = TokenType.Notify;
else if (strToken == "lock")
tokenType = TokenType.Lock;
else if (strToken == "if")
tokenType = TokenType.If;
else if (strToken == "else")
tokenType = TokenType.Else;
else if (strToken == "while")
tokenType = TokenType.While;
else if (strToken == "for")
tokenType = TokenType.For;
else if (strToken == "foreach")
tokenType = TokenType.Foreach;
else if (strToken == "in")
tokenType = TokenType.In;
else if (strToken == "switch")
tokenType = TokenType.Switch;
else if (strToken == "case")
tokenType = TokenType.Case;
else if (strToken == "default")
tokenType = TokenType.Default;
else if (strToken == "break")
tokenType = TokenType.Break;
else if (strToken == "continue")
tokenType = TokenType.Continue;
else if (strToken == "function")
tokenType = TokenType.Function;
else if (strToken == "return")
tokenType = TokenType.Return;
else if (strToken == "thread")
tokenType = TokenType.Thread;
else if (strToken == "null")
tokenType = TokenType.Null;
else if (strToken == "true" || strToken == "false")
tokenType = TokenType.Boolean;
else
tokenType = TokenType.Identifier;
if (tokenType == TokenType.Boolean)
listTokens.Add(new Token(tokenType, strToken == "true",
m_iSourceLine, m_iSourceChar, strSourceLine));
else
listTokens.Add(new Token(tokenType, strToken,
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.String:
if (ch == '\"')
{
listTokens.Add(new Token(TokenType.String, strToken,
m_iSourceLine, m_iSourceChar, strSourceLine));
m_lexState = LexState.Space;
}
else if (ch == '\\')
m_lexState = LexState.StringEscape;
else if (ch == '\r' || ch == '\n')
throw new LexerException("String literal cannot span multiple lines.",
m_iSourceLine, m_iSourceChar, m_listSourceLines[m_iSourceLine]);
else
strToken += ch;
break;
case LexState.StringEscape:
if (ch == '\\' || ch == '\"')
{
strToken += ch;
m_lexState = LexState.String;
}
else if (ch == 't')
{
strToken += '\t';
m_lexState = LexState.String;
}
else if (ch == 'r')
{
strToken += '\r';
m_lexState = LexState.String;
}
else if (ch == 'n')
{
strToken += '\n';
m_lexState = LexState.String;
}
else
throw new LexerException(
"Invalid string escape sequence '\\" + ch + "'.",
m_iSourceLine, m_iSourceChar, m_listSourceLines[m_iSourceLine]);
break;
case LexState.IntegerOrFloat:
if (char.IsDigit(ch))
strToken += ch;
else if (ch == '.')
{
strToken += ch;
m_lexState = LexState.Float;
}
else
{
int iValue = int.Parse(strToken);
listTokens.Add(new Token(TokenType.Integer, iValue,
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
case LexState.Float:
if (char.IsDigit(ch))
strToken += ch;
else
{
float fValue = float.Parse(strToken);
listTokens.Add(new Token(TokenType.Float, fValue,
m_iSourceLine, m_iSourceChar, strSourceLine));
UndoChar();
m_lexState = LexState.Space;
}
break;
default:
throw new LexerException("Unhandled lexer state.");
}
}
if (m_lexState != LexState.Space)
{
throw new LexerException(
"Unexpected end of source reached.");
}
return listTokens;
}
#endregion
}
}
Submit an article or tip