|
#region License
/* **********************************************************************************
* Copyright (c) Roman Ivantsov
* This source code is subject to terms and conditions of the MIT License
* for Irony. A copy of the license can be found in the License.txt file
* at the root of this distribution.
* By using this source code in any fashion, you are agreeing to be bound by the terms of the
* MIT License.
* You must not remove this notice from this software.
* **********************************************************************************/
#endregion
using System;
using System.Collections.Generic;
using System.Text;
namespace Irony.Compiler {
//Adds newLine, indent, unindent tokes to scanner's output stream for languages like Python.
// Scanner ignores new lines and indentations as whitespace; this filter produces these symbols based
// on col/line information in content tokens.
public class CodeOutlineFilter : TokenFilter {
int _prevLine;
Stack<int> _indents = new Stack<int>();
public CodeOutlineFilter(bool trackIndents) {
_trackIndents = trackIndents;
}
public bool TrackIndents {
get {return _trackIndents;}
set {_trackIndents = value;}
} bool _trackIndents = true;
public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens) {
_prevLine = 0;
_indents.Clear();
foreach (Token token in tokens) {
if (token.Terminal == Grammar.Eof) {
yield return CreateNewLine(token.Location); //this is necessary, because grammar rules reference newLine terminator
//unindent all buffered indents
if (_trackIndents)
foreach(int i in _indents)
yield return CreateDedent(token.Location);
_indents.Clear();
yield return token;
yield break;
}//if Eof
//Now deal with normal, non-EOF tokens
//We intercept only content tokens on new lines
if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine) {
yield return token;
continue;
}
//if we are here, we have content token on new line; produce newLine token and possibly indents
yield return CreateNewLine(token.Location);
_prevLine = token.Location.Line;
if (!_trackIndents) {
yield return token;
continue;
}
//Now take care of indents
int currIndent = token.Location.Column;
int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek();
if (currIndent > prevIndent) {
_indents.Push(currIndent);
yield return CreateIndent(token.Location);
} else if (currIndent < prevIndent) {
//produce one or more dedent tokens while popping indents from stack
while (_indents.Peek() > currIndent) {
_indents.Pop();
yield return CreateDedent(token.Location);
}
if (_indents.Peek() != currIndent) {
yield return Grammar.CreateErrorToken(token.Location, "Invalid dedent level, no previous matching indent found.");
//TODO: add error recovery here
}
}//else if currIndent < prevIndent
yield return token;
} //foreach token
}//method
private Token CreateNewLine(SourceLocation location) {
return new Token(Grammar.NewLine, location, "<LF>");
}
private Token CreateIndent(SourceLocation location) {
return new Token(Grammar.Indent, location, "<Indent>");
}
private Token CreateDedent(SourceLocation location) {
return new Token(Grammar.Dedent, location, "<Dedent>");
}
}//class
}
|
By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.
If a file you wish to view isn't highlighted, and is a text file (not binary), please
let us know and we'll add colourisation support for it.
25 years of professional experience. .NET/c#, databases, security.
Currently Senior Security Engineer, Cloud Security, Microsoft