|
#region License
/* **********************************************************************************
* Copyright (c) Roman Ivantsov
* This source code is subject to terms and conditions of the MIT License
* for Irony. A copy of the license can be found in the License.txt file
* at the root of this distribution.
* By using this source code in any fashion, you are agreeing to be bound by the terms of the
* MIT License.
* You must not remove this notice from this software.
* **********************************************************************************/
#endregion
using System;
using System.Collections.Generic;
using System.Text;
namespace Irony.Compiler {
//Scanner class. The Scanner's function is to transform a stream of characters into bigger aggregates/words or lexemes,
// like identifier, number, literal, etc.
public class Scanner {
public Scanner(GrammarData data) {
_data = data;
}
#region Fields: _data, _source, _context
GrammarData _data;
SourceFile _source;
CompilerContext _context;
#endregion
#region Events: TokenCreated
//Note that scanner's output stream may not contain all tokens received by parser. Additional tokens
// may be generated by intermediate token filters. To listen to token stream at parser input,
// use Parser's TokenReceived event.
public event EventHandler<TokenEventArgs> TokenCreated;
TokenEventArgs _tokenArgs = new TokenEventArgs(null);
protected void OnTokenCreated(Token token) {
if (TokenCreated == null) return;
_tokenArgs.Token = token;
TokenCreated(this, _tokenArgs);
}
#endregion
Token _currentToken;
public IEnumerable<Token> BeginScan(CompilerContext context, SourceFile source) {
_context = context;
_source = source;
_source.Reset();
while (true) {
_currentToken = ReadToken();
if (TokenCreated != null)
OnTokenCreated(_currentToken);
//if (tkn.Terminal.Category != TerminalCategory.Comment)
yield return _currentToken;
if (_currentToken.Terminal == Grammar.Eof)
yield break;
}//while
}// method
private Token ReadToken() {
string wspace = _data.Grammar.WhitespaceChars;
_source.SetNextTokenStart(wspace);
//Check for EOF
if (_source.EOF())
return new Token(Grammar.Eof, _source.TokenStart, string.Empty, Grammar.Eof.Name);
//Find matching terminal
TerminalList terms = SelectTerminals(_source.CurrentChar);
Token result = null;
int resultEndPos = 0;
foreach (Terminal term in terms) {
Token token = term.TryMatch(_context, _source);
if (token != null && (result == null || _source.Position > resultEndPos)) {
result = token;
resultEndPos = _source.Position;
}
_source.Position = _source.TokenStart.Position;
}
if (result != null) {
_source.Position = resultEndPos;
} else {
result = Grammar.CreateErrorToken(_source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar);
//Primitive error recovery - skip until whitespace.
while (wspace.IndexOf(_source.CurrentChar) < 0)
_source.Position++;
}
return result;
}//method
public TerminalList SelectTerminals(char current) {
TerminalList result;
if (_data.TerminalsLookup.TryGetValue(current, out result))
return result;
else
return _data.NoPrefixTerminals;
}//Select
public override string ToString() {
return _source.ToString(); //show 30 chars starting from current position
}
}//class
}//namespace
|
By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.
If a file you wish to view isn't highlighted, and is a text file (not binary), please
let us know and we'll add colourisation support for it.
25 years of professional experience. .NET/c#, databases, security.
Currently Senior Security Engineer, Cloud Security, Microsoft