Tokenizer and analyzer package supporting precedence prioritized rules

#ifdef _DEBUG
#include "cxTokenizerTextInputStream.h"

bool	cxTokenizer::fCheckValid() const
	{
	if(!m_tcContextInfo.fCheckValid())
		{
		TRACE(_T("cxTokenizer::fCheckValid() - context information corrupted.\n"));
		return false;
		}

	if(m_ptisData!=NULL)
		{
		if(!m_ptisData->fCheckValid())
			{
			TRACE(_T("cxTokenizer::fCheckValid() - input stream is not valid.\n"));
			return false;
			}
		}

	if(m_ptlReceiver==NULL)
		{
		TRACE(_T("cxTokenizer::fCheckValid() - receiver is NULL.\n"));
		return false;
		}

	if(!m_ptlReceiver->fCheckValid())
		{
		TRACE(_T("cxTokenizer::fCheckValid() - receiver is not valid.\n"));
		return false;
		}

	if(m_ptmLexxerMap==NULL)
		{
		TRACE(_T("cxTokenizer::fCheckValid() - lexxer map is NULL.\n"));
		return false;
		}

	if(!m_ptmLexxerMap->fCheckValid())
		{
		TRACE(_T("cxTokenizer::fCheckValid() - lexxer map is not valid.\n"));
		return false;
		}

	return true;
	}

bool	cxTokenizer::fRunDiagnostics()
	{
	class	cxTestListener : public cxTokenizerListener
	{
	// Attributes
	public:
		int		nTokenNumber;
		bool	fValid;

	// Operations
	public:
		virtual	bool			fCheckValid() const	{ return true; };
		virtual bool			fShouldDelete() const { return false; };
		virtual	void			vRegisterToken(const std::tstring& strToken, cxTokenizerTokenRule* pptrRule)
			{
			TRACE(_T("Token #%02d: %s\n"),nTokenNumber, strToken.data());
			switch(nTokenNumber)
				{
				case 0:
					if(strToken.compare(_T("test"))!=0 ||
						pptrRule==NULL)
						fValid=false;
					break;
				case 1:
					if(strToken.compare(_T("--"))!=0)
						fValid=false;
					break;
				case 2:
					if(strToken.compare(_T("-"))!=0)
						fValid=false;
					break;
				case 3:
					if(	strToken.compare(_T("test1"))!=0 ||
						pptrRule!=NULL)
						fValid=false;
					break;
				case 4:
					if(strToken.compare(_T("--"))!=0)
						fValid=false;
					break;
				case 5:
					if(	strToken.compare(_T("1234"))!=0 || 
						pptrRule->fIsFlagSet(ttrf_character_rule))
						fValid=false;
					break;
				case 6:
					if(strToken.compare(_T("-"))!=0)
						fValid=false;
					break;
				case 7:
					if(strToken.compare(_T("test"))!=0)
						fValid=false;
					break;
				};
			nTokenNumber++;
			};
	};

	std::tstringstream	strm(
		_T("[seperators]\n")
		_T("-\n")
		_T("--\n")
		_T("[tokens]\n")
		_T("test\n")
		_T("[rules]\n")
		_T("strings\n")
		_T("numbers\n") );

/*	cxTokenizerMap				map;
	cxTokenizerTextInputStream	tis(_T("test---test1--1234-\"test\""));
	cooTestListener				lis;
	cxTokenizer					lexxer(&tis,&map,&lis);

	lis.nTokenNumber	=0;
	lis.fValid			=true;

	// Initialize map
	map.vLoadFromStream(strm);

	while(!tis.fIsEofReached())
		lexxer.vParseCharacter();

	return lis.fValid?lexxer.fCheckValid():false;*/
	return false;
	}
#endif