// Generated by TinyPG v1.2 available at www.codeproject.com
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
namespace TinyPG.Highlighter
{
#region Scanner
public partial class Scanner
{
public string Input;
public int StartPos = 0;
public int EndPos = 0;
public int CurrentLine;
public int CurrentColumn;
public int CurrentPosition;
public List<Token> Skipped; // tokens that were skipped
public Dictionary<TokenType, Regex> Patterns;
private Token LookAheadToken;
private List<TokenType> Tokens;
private List<TokenType> SkipList; // tokens to be skipped
public Scanner()
{
Regex regex;
Patterns = new Dictionary<TokenType, Regex>();
Tokens = new List<TokenType>();
LookAheadToken = null;
SkipList = new List<TokenType>();
regex = new Regex(@"\s+", RegexOptions.Compiled);
Patterns.Add(TokenType.WHITESPACE, regex);
Tokens.Add(TokenType.WHITESPACE);
regex = new Regex(@"^$", RegexOptions.Compiled);
Patterns.Add(TokenType.EOF, regex);
Tokens.Add(TokenType.EOF);
regex = new Regex(@"//[^\n]*\n", RegexOptions.Compiled);
Patterns.Add(TokenType.GRAMMARCOMMENTLINE, regex);
Tokens.Add(TokenType.GRAMMARCOMMENTLINE);
regex = new Regex(@"/\*([^*]+|\*[^/])+(\*/)?", RegexOptions.Compiled);
Patterns.Add(TokenType.GRAMMARCOMMENTBLOCK, regex);
Tokens.Add(TokenType.GRAMMARCOMMENTBLOCK);
regex = new Regex(@"@?\""(\""\""|[^\""])*(""|\n)", RegexOptions.Compiled);
Patterns.Add(TokenType.DIRECTIVESTRING, regex);
Tokens.Add(TokenType.DIRECTIVESTRING);
regex = new Regex(@"^(@TinyPG|@Parser|@Scanner|@Grammar|@ParseTree|@TextHighlighter)", RegexOptions.Compiled);
Patterns.Add(TokenType.DIRECTIVEKEYWORD, regex);
Tokens.Add(TokenType.DIRECTIVEKEYWORD);
regex = new Regex(@"^(@|(%[^>])|=|"")+?", RegexOptions.Compiled);
Patterns.Add(TokenType.DIRECTIVESYMBOL, regex);
Tokens.Add(TokenType.DIRECTIVESYMBOL);
regex = new Regex(@"[^%@=""]+", RegexOptions.Compiled);
Patterns.Add(TokenType.DIRECTIVENONKEYWORD, regex);
Tokens.Add(TokenType.DIRECTIVENONKEYWORD);
regex = new Regex(@"<%", RegexOptions.Compiled);
Patterns.Add(TokenType.DIRECTIVEOPEN, regex);
Tokens.Add(TokenType.DIRECTIVEOPEN);
regex = new Regex(@"%>", RegexOptions.Compiled);
Patterns.Add(TokenType.DIRECTIVECLOSE, regex);
Tokens.Add(TokenType.DIRECTIVECLOSE);
regex = new Regex(@"[^\[\]]", RegexOptions.Compiled);
Patterns.Add(TokenType.ATTRIBUTESYMBOL, regex);
Tokens.Add(TokenType.ATTRIBUTESYMBOL);
regex = new Regex(@"^(Skip|Color)", RegexOptions.Compiled);
Patterns.Add(TokenType.ATTRIBUTEKEYWORD, regex);
Tokens.Add(TokenType.ATTRIBUTEKEYWORD);
regex = new Regex(@"[^\(\)\]\n\s]+", RegexOptions.Compiled);
Patterns.Add(TokenType.ATTRIBUTENONKEYWORD, regex);
Tokens.Add(TokenType.ATTRIBUTENONKEYWORD);
regex = new Regex(@"\[\s*", RegexOptions.Compiled);
Patterns.Add(TokenType.ATTRIBUTEOPEN, regex);
Tokens.Add(TokenType.ATTRIBUTEOPEN);
regex = new Regex(@"\s*\]\s*", RegexOptions.Compiled);
Patterns.Add(TokenType.ATTRIBUTECLOSE, regex);
Tokens.Add(TokenType.ATTRIBUTECLOSE);
regex = new Regex(@"^(abstract|as|base|break|case|catch|checked|class|const|continue|decimal|default|delegate|double|do|else|enum|event|explicit|extern|false|finally|fixed|float|foreach|for|get|goto|if|implicit|interface|internal|int|in|is|lock|namespace|new|null|object|operator|out|override|params|partial|private|protected|public|readonly|ref|return|sealed|set|sizeof|stackalloc|static|struct|switch|this|throw|true|try|typeof|unchecked|unsafe|ushort|using|virtual|void|volatile|while)", RegexOptions.Compiled);
Patterns.Add(TokenType.CS_KEYWORD, regex);
Tokens.Add(TokenType.CS_KEYWORD);
regex = new Regex(@"^(AddHandler|AddressOf|Alias|AndAlso|And|Ansi|Assembly|As|Auto|Boolean|ByRef|Byte|ByVal|Call|Case|Catch|CBool|CByte|CChar|CDate|CDec|CDbl|Char|CInt|Class|CLng|CObj|Const|CShort|CSng|CStr|CType|Date|Decimal|Declare|Default|Delegate|Dim|DirectCast|Double|Do|Each|ElseIf|Else|End|Enum|Erase|Error|Event|Exit|False|Finally|For|Friend|Function|GetType|Get|GoSub|GoTo|Handles|If|Implements|Imports|Inherits|Integer|Interface|In|IsNot|Is|Let|Lib|Like|Long|Loop|Me|Mod|Module|MustInherit|MustOverride|MyBase|MyClass|Namespace|New|Next|Nothing|NotInheritable|NotOverridable|Not|Object|On|Optional|Option|OrElse|Or|Overloads|Overridable|Overrides|ParamArray|Preserve|Private|Property|Protected|Public|RaiseEvent|ReadOnly|ReDim|REM|RemoveHandler|Resume|Return|Select|Set|Shadows|Shared|Short|Single|Static|Step|Stop|String|Structure|Sub|SyncLock|Then|Throw|To|True|Try|TypeOf|Unicode|Until|Variant|When|While|With|WithEvents|WriteOnly|Xor|Source)", RegexOptions.Compiled);
Patterns.Add(TokenType.VB_KEYWORD, regex);
Tokens.Add(TokenType.VB_KEYWORD);
regex = new Regex(@"^(abstract|as|base|break|case|catch|checked|class|const|continue|decimal|default|delegate|double|do|else|enum|event|explicit|extern|false|finally|fixed|float|foreach|for|get|goto|if|implicit|interface|internal|int|in|is|lock|namespace|new|null|object|operator|out|override|params|partial|private|protected|public|readonly|ref|return|sealed|set|sizeof|stackalloc|static|struct|switch|this|throw|true|try|typeof|unchecked|unsafe|ushort|using|virtual|void|volatile|while)", RegexOptions.Compiled);
Patterns.Add(TokenType.DOTNET_KEYWORD, regex);
Tokens.Add(TokenType.DOTNET_KEYWORD);
regex = new Regex(@"^(Array|AttributeTargets|AttributeUsageAttribute|Attribute|BitConverter|Boolean|Buffer|Byte|Char|CharEnumerator|CLSCompliantAttribute|ConsoleColor|ConsoleKey|ConsoleKeyInfo|ConsoleModifiers|ConsoleSpecialKey|Console|ContextBoundObject|ContextStaticAttribute|Converter|Convert|DateTimeKind|DateTimeOffset|DateTime|DayOfWeek|DBNull|Decimal|Delegate|Double|Enum|Environment.SpecialFolder|EnvironmentVariableTarget|Environment|EventArgs|EventHandler|Exception|FlagsAttribute|GCCollectionMode|GC|Guid|ICloneable|IComparable|IConvertible|ICustomFormatter|IDisposable|IEquatable|IFormatProvider|IFormattable|IndexOutOfRangeException|InsufficientMemoryException|Int16|Int32|Int64|IntPtr|InvalidCastException|InvalidOperationException|InvalidProgramException|MarshalByRefObject|Math|MidpointRounding|NotFiniteNumberException|NotImplementedException|NotSupportedException|Nullable|NullReferenceException|ObjectDisposedException|Object|ObsoleteAttribute|OperatingSystem|OutOfMemoryException|OverflowException|ParamArrayAttribute|PlatformID|PlatformNotSupportedException|Predicate|Random|SByte|SerializableAttribute|Single|StackOverflowException|StringComparer|StringComparison|StringSplitOptions|String|SystemException|TimeSpan|TimeZone|TypeCode|TypedReference|TypeInitializationException|Type|UInt16|UInt32|UInt64|UIntPtr|UnauthorizedAccessException|UnhandledExceptionEventArgs|UnhandledExceptionEventHandler|ValueType|Void|WeakReference|Comparer|Dictionary|EqualityComparer|ICollection|IComparer|IDictionary|IEnumerable|IEnumerator|IEqualityComparer|IList|KeyNotFoundException|KeyValuePair|List|ASCIIEncoding|Decoder|DecoderExceptionFallback|DecoderExceptionFallbackBuffer|DecoderFallback|DecoderFallbackBuffer|DecoderFallbackException|DecoderReplacementFallback|DecoderReplacementFallbackBuffer|EncoderExceptionFallback|EncoderExceptionFallbackBuffer|EncoderFallback|EncoderFallbackBuffer|EncoderFallbackException|EncoderReplacementFallback|EncoderReplacementFallbackBuffer|Encoder|EncodingInfo|Encoding|NormalizationForm|StringBuilder|UnicodeEncoding|UTF32Encoding|UTF7Encoding|UTF8Encoding)", RegexOptions.Compiled);
Patterns.Add(TokenType.DOTNET_TYPES, regex);
Tokens.Add(TokenType.DOTNET_TYPES);
regex = new Regex(@"//[^\n]*\n", RegexOptions.Compiled);
Patterns.Add(TokenType.CS_COMMENTLINE, regex);
Tokens.Add(TokenType.CS_COMMENTLINE);
regex = new Regex(@"/\*([^*]+|\*[^/])+(\*/)?", RegexOptions.Compiled);
Patterns.Add(TokenType.CS_COMMENTBLOCK, regex);
Tokens.Add(TokenType.CS_COMMENTBLOCK);
regex = new Regex(@"[^}]", RegexOptions.Compiled);
Patterns.Add(TokenType.CS_SYMBOL, regex);
Tokens.Add(TokenType.CS_SYMBOL);
regex = new Regex(@"([^""\n\s/;.}\(\)\[\]]|/[^/*]|}[^;])+", RegexOptions.Compiled);
Patterns.Add(TokenType.CS_NONKEYWORD, regex);
Tokens.Add(TokenType.CS_NONKEYWORD);
regex = new Regex(@"@?[""]([""][""]|[^\""\n])*[""]?", RegexOptions.Compiled);
Patterns.Add(TokenType.CS_STRING, regex);
Tokens.Add(TokenType.CS_STRING);
regex = new Regex(@"'[^\n]*\n", RegexOptions.Compiled);
Patterns.Add(TokenType.VB_COMMENTLINE, regex);
Tokens.Add(TokenType.VB_COMMENTLINE);
regex = new Regex(@"REM[^\n]*\n", RegexOptions.Compiled);
Patterns.Add(TokenType.VB_COMMENTBLOCK, regex);
Tokens.Add(TokenType.VB_COMMENTBLOCK);
regex = new Regex(@"[^}]", RegexOptions.Compiled);
Patterns.Add(TokenType.VB_SYMBOL, regex);
Tokens.Add(TokenType.VB_SYMBOL);
regex = new Regex(@"([^""\n\s/;.}\(\)\[\]]|/[^/*]|}[^;])+", RegexOptions.Compiled);
Patterns.Add(TokenType.VB_NONKEYWORD, regex);
Tokens.Add(TokenType.VB_NONKEYWORD);
regex = new Regex(@"@?[""]([""][""]|[^\""\n])*[""]?", RegexOptions.Compiled);
Patterns.Add(TokenType.VB_STRING, regex);
Tokens.Add(TokenType.VB_STRING);
regex = new Regex(@"//[^\n]*\n", RegexOptions.Compiled);
Patterns.Add(TokenType.DOTNET_COMMENTLINE, regex);
Tokens.Add(TokenType.DOTNET_COMMENTLINE);
regex = new Regex(@"/\*([^*]+|\*[^/])+(\*/)?", RegexOptions.Compiled);
Patterns.Add(TokenType.DOTNET_COMMENTBLOCK, regex);
Tokens.Add(TokenType.DOTNET_COMMENTBLOCK);
regex = new Regex(@"[^}]", RegexOptions.Compiled);
Patterns.Add(TokenType.DOTNET_SYMBOL, regex);
Tokens.Add(TokenType.DOTNET_SYMBOL);
regex = new Regex(@"([^""\n\s/;.}\[\]]|/[^/*]|}[^;])+", RegexOptions.Compiled);
Patterns.Add(TokenType.DOTNET_NONKEYWORD, regex);
Tokens.Add(TokenType.DOTNET_NONKEYWORD);
regex = new Regex(@"@?[""]([""][""]|[^\""\n])*[""]?", RegexOptions.Compiled);
Patterns.Add(TokenType.DOTNET_STRING, regex);
Tokens.Add(TokenType.DOTNET_STRING);
regex = new Regex(@"\{", RegexOptions.Compiled);
Patterns.Add(TokenType.CODEBLOCKOPEN, regex);
Tokens.Add(TokenType.CODEBLOCKOPEN);
regex = new Regex(@"\};", RegexOptions.Compiled);
Patterns.Add(TokenType.CODEBLOCKCLOSE, regex);
Tokens.Add(TokenType.CODEBLOCKCLOSE);
regex = new Regex(@"(Start)", RegexOptions.Compiled);
Patterns.Add(TokenType.GRAMMARKEYWORD, regex);
Tokens.Add(TokenType.GRAMMARKEYWORD);
regex = new Regex(@"->", RegexOptions.Compiled);
Patterns.Add(TokenType.GRAMMARARROW, regex);
Tokens.Add(TokenType.GRAMMARARROW);
regex = new Regex(@"([^""\[\n\s/<{]|/[^/*]|<[^%])+", RegexOptions.Compiled);
Patterns.Add(TokenType.GRAMMARNONKEYWORD, regex);
Tokens.Add(TokenType.GRAMMARNONKEYWORD);
regex = new Regex(@"@?[""]([""][""]|[^\""\n])*[""]?", RegexOptions.Compiled);
Patterns.Add(TokenType.GRAMMARSTRING, regex);
Tokens.Add(TokenType.GRAMMARSTRING);
}
public void Init(string input)
{
this.Input = input;
StartPos = 0;
EndPos = 0;
CurrentLine = 0;
CurrentColumn = 0;
CurrentPosition = 0;
Skipped = new List<Token>();
LookAheadToken = null;
}
public Token GetToken(TokenType type)
{
Token t = new Token(this.StartPos, this.EndPos);
t.Type = type;
return t;
}
/// <summary>
/// executes a lookahead of the next token
/// and will advance the scan on the input string
/// </summary>
/// <returns></returns>
public Token Scan(params TokenType[] expectedtokens)
{
Token tok = LookAhead(expectedtokens); // temporarely retrieve the lookahead
LookAheadToken = null; // reset lookahead token, so scanning will continue
StartPos = tok.EndPos;
EndPos = tok.EndPos; // set the tokenizer to the new scan position
return tok;
}
/// <summary>
/// returns token with longest best match
/// </summary>
/// <returns></returns>
public Token LookAhead(params TokenType[] expectedtokens)
{
int i;
int startpos = StartPos;
Token tok = null;
List<TokenType> scantokens;
// this prevents double scanning and matching
// increased performance
if (LookAheadToken != null
&& LookAheadToken.Type != TokenType._UNDETERMINED_
&& LookAheadToken.Type != TokenType._NONE_) return LookAheadToken;
// if no scantokens specified, then scan for all of them (= backward compatible)
if (expectedtokens.Length == 0)
scantokens = Tokens;
else
{
scantokens = new List<TokenType>(expectedtokens);
scantokens.AddRange(SkipList);
}
do
{
int len = -1;
TokenType index = (TokenType)int.MaxValue;
string input = Input.Substring(startpos);
tok = new Token(startpos, EndPos);
for (i = 0; i < scantokens.Count; i++)
{
Regex r = Patterns[scantokens[i]];
Match m = r.Match(input);
if (m.Success && m.Index == 0 && ((m.Length > len) || (scantokens[i] < index && m.Length == len )))
{
len = m.Length;
index = scantokens[i];
}
}
if (index >= 0 && len >= 0)
{
tok.EndPos = startpos + len;
tok.Text = Input.Substring(tok.StartPos, len);
tok.Type = index;
}
else
{
if (tok.StartPos < tok.EndPos - 1)
tok.Text = Input.Substring(tok.StartPos, 1);
}
if (SkipList.Contains(tok.Type))
{
startpos = tok.EndPos;
Skipped.Add(tok);
}
}
while (SkipList.Contains(tok.Type));
LookAheadToken = tok;
return tok;
}
}
#endregion
#region Token
public enum TokenType
{
//Non terminal tokens:
_NONE_ = 0,
_UNDETERMINED_= 1,
//Non terminal tokens:
Start = 2,
DirectiveBlock= 3,
CommentBlock= 4,
GrammarBlock= 5,
AttributeBlock= 6,
CodeBlock= 7,
//Terminal tokens:
WHITESPACE= 8,
EOF = 9,
GRAMMARCOMMENTLINE= 10,
GRAMMARCOMMENTBLOCK= 11,
DIRECTIVESTRING= 12,
DIRECTIVEKEYWORD= 13,
DIRECTIVESYMBOL= 14,
DIRECTIVENONKEYWORD= 15,
DIRECTIVEOPEN= 16,
DIRECTIVECLOSE= 17,
ATTRIBUTESYMBOL= 18,
ATTRIBUTEKEYWORD= 19,
ATTRIBUTENONKEYWORD= 20,
ATTRIBUTEOPEN= 21,
ATTRIBUTECLOSE= 22,
CS_KEYWORD= 23,
VB_KEYWORD= 24,
DOTNET_KEYWORD= 25,
DOTNET_TYPES= 26,
CS_COMMENTLINE= 27,
CS_COMMENTBLOCK= 28,
CS_SYMBOL= 29,
CS_NONKEYWORD= 30,
CS_STRING= 31,
VB_COMMENTLINE= 32,
VB_COMMENTBLOCK= 33,
VB_SYMBOL= 34,
VB_NONKEYWORD= 35,
VB_STRING= 36,
DOTNET_COMMENTLINE= 37,
DOTNET_COMMENTBLOCK= 38,
DOTNET_SYMBOL= 39,
DOTNET_NONKEYWORD= 40,
DOTNET_STRING= 41,
CODEBLOCKOPEN= 42,
CODEBLOCKCLOSE= 43,
GRAMMARKEYWORD= 44,
GRAMMARARROW= 45,
GRAMMARNONKEYWORD= 46,
GRAMMARSTRING= 47
}
public class Token
{
private int startpos;
private int endpos;
private string text;
private object value;
public int StartPos {
get { return startpos;}
set { startpos = value; }
}
public int Length {
get { return endpos - startpos;}
}
public int EndPos {
get { return endpos;}
set { endpos = value; }
}
public string Text {
get { return text;}
set { text = value; }
}
public object Value {
get { return value;}
set { this.value = value; }
}
public TokenType Type;
public Token()
: this(0, 0)
{
}
public Token(int start, int end)
{
Type = TokenType._UNDETERMINED_;
startpos = start;
endpos = end;
Text = ""; // must initialize with empty string, may cause null reference exceptions otherwise
Value = null;
}
public void UpdateRange(Token token)
{
if (token.StartPos < startpos) startpos = token.StartPos;
if (token.EndPos > endpos) endpos = token.EndPos;
}
public override string ToString()
{
if (Text != null)
return Type.ToString() + " '" + Text + "'";
else
return Type.ToString();
}
}
#endregion
}