Irony - .NET Compiler Construction Kit

#region License
/* **********************************************************************************
 * Copyright (c) Roman Ivantsov
 * This source code is subject to terms and conditions of the MIT License
 * for Irony. A copy of the license can be found in the License.txt file
 * at the root of this distribution. 
 * By using this source code in any fashion, you are agreeing to be bound by the terms of the 
 * MIT License.
 * You must not remove this notice from this software.
 * **********************************************************************************/
#endregion

using System;
using System.Collections.Generic;
using System.Text;

namespace Irony.Compiler {

  //Scanner class. The Scanner's function is to transform a stream of characters into bigger aggregates/words or lexemes, 
  // like identifier, number, literal, etc. 

  public class Scanner  {
    public Scanner(GrammarData data)  {
      _data = data;
    }

    #region Fields: _data, _source, _context
    GrammarData _data;
    SourceFile  _source;
    CompilerContext  _context;
    #endregion

    #region Events: TokenCreated
    //Note that scanner's output stream may not contain all tokens received by parser. Additional tokens
    // may be generated by intermediate token filters. To listen to token stream at parser input, 
    // use Parser's TokenReceived event. 
    public event EventHandler<TokenEventArgs> TokenCreated;
    TokenEventArgs _tokenArgs = new TokenEventArgs(null);

    protected void OnTokenCreated(Token token) {
      if (TokenCreated == null) return;
      _tokenArgs.Token = token;
      TokenCreated(this, _tokenArgs);
    }
    #endregion

    Token _currentToken;
    public IEnumerable<Token> BeginScan(CompilerContext context, SourceFile source) {
      _context = context;
      _source = source;

      _source.Reset();
      while (true) {
        _currentToken = ReadToken();
        if (TokenCreated != null)
          OnTokenCreated(_currentToken);
        //if (tkn.Terminal.Category != TerminalCategory.Comment)
        yield return _currentToken;
        if (_currentToken.Terminal == Grammar.Eof)
          yield break;
      }//while
    }// method

    private Token ReadToken() {
      string wspace = _data.Grammar.WhitespaceChars;
      _source.SetNextTokenStart(wspace);
      //Check for EOF
      if (_source.EOF()) 
        return new Token(Grammar.Eof, _source.TokenStart, string.Empty, Grammar.Eof.Name);
      //Find matching terminal
      TerminalList terms = SelectTerminals(_source.CurrentChar);
      Token result = null;
      int resultEndPos = 0;
      foreach (Terminal term in terms) {
        Token token = term.TryMatch(_context, _source);
        if (token != null && (result == null || _source.Position > resultEndPos)) {
          result = token;
          resultEndPos = _source.Position;
        }
        _source.Position = _source.TokenStart.Position;
      }
      if (result != null) {
        _source.Position = resultEndPos;
      } else {
        result = Grammar.CreateErrorToken(_source.TokenStart, "Invalid character: '{0}'", _source.CurrentChar);
        //Primitive error recovery - skip until whitespace.
        while (wspace.IndexOf(_source.CurrentChar) < 0)
          _source.Position++;
      }
      return result;
    }//method

    public TerminalList SelectTerminals(char current) {
      TerminalList result;
      if (_data.TerminalsLookup.TryGetValue(current, out result))
        return result;
      else
        return _data.NoPrefixTerminals;
    }//Select
    

    public override string ToString() {
      return _source.ToString(); //show 30 chars starting from current position
    }

  }//class

}//namespace