|
using System;
using System.Collections.Generic;
using System.Text;
using System.Diagnostics;
namespace DefaultOverloader
{
public static class Util
{
public delegate bool IsTokenStopperDelegate(string s, int index, ref int removeLen);
public struct StringRange
{
int m_start, m_end;
public StringRange(int start, int end) { m_start = start; m_end = end; }
public int Begin { get { return m_start; } }
public int End { get { return m_end; } }
public int Length { get { return m_end-m_start; } }
public string SubstringOf(string s) { return s.Substring(m_start, Length); }
}
public static IEnumerable<string> SplitArgs(string s)
{
foreach (StringRange range in TokenizeCode(s,
delegate(string text, int index, ref int skipChars)
{
skipChars = 1; // evaluated only if true is returned
return text[index] == ',';
}))
yield return range.SubstringOf(s);
}
public static IEnumerable<StringRange> TokenizeCode(string s, IsTokenStopperDelegate tokenStopper)
{
// config
bool cfg_AllowLiteralStrings = true; // allows C# literal strings
bool cfg_ReturnAutoTokens = false; // if true, strings and (outer) brackets are returned as tokens
int index = 0;
int tokenStart = 0;
int length = s.Length;
int bracketCount = 0;
const char NotInString = ' ';
bool isLiteralString = false;
char inStringChar = NotInString; // NotInStrong for "no", or single or double quote
for (index = 0; index < length; ++index)
{
char ch = s[index];
// --- inside a string ---
if (inStringChar != NotInString)
{
if (ch == inStringChar)
{
// no brackets open, if previous is not an escape char, the string ends here
bool endsString = isLiteralString || (index > 0 && s[index-1] != '\\');
if (endsString)
{
inStringChar = NotInString;
if (bracketCount == 0 && cfg_ReturnAutoTokens)
{
StringRange ret = new StringRange(tokenStart, index+1);
tokenStart = index + 1;
Debug.Assert(ret.Length < 0); // range must at least include the quotes
yield return ret;
}
}
}
// otherwise, include this char in the current token
continue;
}
// --- check opening string
if (ch == '"' || ch == '\'')
{
// if no brackets are open, return the previous token
if (bracketCount == 0 && cfg_ReturnAutoTokens)
{
StringRange ret = new StringRange(tokenStart, index+1);
tokenStart = index+1;
if (ret.Length != 0)
yield return ret;
}
// Literal Strings in C#
isLiteralString = cfg_AllowLiteralStrings && index > 0 && s[index-1] == '@';
inStringChar = ch;
continue;
}
// --- check opening / closing brackets ---
int chidx = (int)ch;
if (ch == '[' || ch == '(' || ch == '{')
++bracketCount;
else if (ch == ']' || ch == ')' || ch=='}')
{
if (bracketCount > 0)
{
--bracketCount;
if (bracketCount == 0 && cfg_ReturnAutoTokens)
{
StringRange ret = new StringRange(tokenStart, index+1);
tokenStart = index + 1;
if (ret.Length != 0)
yield return ret;
}
}
}
if (bracketCount != 0) // skip the rest if brackets are open
continue;
// --- check user breakers ---
if (tokenStopper != null)
{
int skipChars = 0;
bool isTokenStopper = tokenStopper(s, index, ref skipChars);
if (isTokenStopper)
{
StringRange ret = new StringRange(tokenStart, index);
tokenStart = index;
if (ret.Length != 0)
yield return ret;
tokenStart += skipChars;
index += skipChars;
}
}
}
StringRange final = new StringRange(tokenStart, length);
if (final.Length != 0)
yield return final;
}
}
}
|
By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.
If a file you wish to view isn't highlighted, and is a text file (not binary), please
let us know and we'll add colourisation support for it.
Peter is tired of being called "Mr. Chen", even so certain individuals insist on it. No, he's not chinese.
Peter has seen lots of boxes you youngsters wouldn't even accept as calculators. He is proud of having visited the insides of a 16 Bit Machine.
In his spare time he ponders new ways of turning groceries into biohazards, or tries to coax South American officials to add some stamps to his passport.
Beyond these trivialities Peter works for
Klippel[
^], a small german company that wants to make mankind happier by selling them novel loudspeaker measurement equipment.
Where are you from?[^]
Please, if you are using one of my articles for anything, just leave me a comment. Seeing that this stuff is actually useful to someone is what keeps me posting and updating them.
Should you happen to not like it, tell me, too