Fast tokenizer

#ifdef _DEBUG
#include "cooLexxerTextInputStream.h"

bool	cooLexxer::fCheckValid() const
	{

	LPCTSTR	lpszCause = NULL;

	if(!m_lcContextInfo.fCheckValid())
		{
		TRACE(_T("cooLexxer::fCheckValid() - context information corrupted.\n"));
		return false;
		}

	if(m_plisData==NULL)
		{
		TRACE(_T("cooLexxer::fCheckValid() - input stream is NULL.\n"));
		return false;
		}

	if(!m_plisData->fCheckValid())
		{
		TRACE(_T("cooLexxer::fCheckValid() - input stream is not valid.\n"));
		return false;
		}

	if(m_pllReceiver==NULL)
		{
		TRACE(_T("cooLexxer::fCheckValid() - receiver is NULL.\n"));
		return false;
		}

	if(!m_pllReceiver->fCheckValid())
		{
		TRACE(_T("cooLexxer::fCheckValid() - receiver is not valid.\n"));
		return false;
		}

	if(m_plmLexxerMap==NULL)
		{
		TRACE(_T("cooLexxer::fCheckValid() - lexxer map is NULL.\n"));
		return false;
		}

	if(!m_plmLexxerMap->fCheckValid())
		{
		TRACE(_T("cooLexxer::fCheckValid() - lexxer map is not valid.\n"));
		return false;
		}

	return true;
	}

bool	cooLexxer::fRunDiagnostics()
	{
	class	cooTestListener : public cooLexxerListener
	{
	// Attributes
	public:
		int		nTokenNumber;
		bool	fValid;

	// Operations
	public:
		virtual	bool			fCheckValid() const	{ return true; };
		virtual bool			fShouldDelete() const { return false; };
		virtual	void			vRegisterToken(const std::tstring& strToken, const cooLexxerTokenRule* pptrRule)
			{
			TRACE(_T("Token #%02d: %s\n"),nTokenNumber, strToken.data());
			switch(nTokenNumber)
				{
				case 0:
					if(strToken.compare(_T("test"))!=0 ||
						pptrRule==NULL)
						fValid=false;
					break;
				case 1:
					if(strToken.compare(_T("--"))!=0)
						fValid=false;
					break;
				case 2:
					if(strToken.compare(_T("-"))!=0)
						fValid=false;
					break;
				case 3:
					if(	strToken.compare(_T("test1"))!=0 ||
						pptrRule!=NULL)
						fValid=false;
					break;
				case 4:
					if(strToken.compare(_T("--"))!=0)
						fValid=false;
					break;
				case 5:
					if(	strToken.compare(_T("1234"))!=0 || 
						pptrRule->fIsFlagSet(ooltrf_character_rule))
						fValid=false;
					break;
				case 6:
					if(strToken.compare(_T("-"))!=0)
						fValid=false;
					break;
				case 7:
					if(strToken.compare(_T("test"))!=0)
						fValid=false;
					break;
				};
			nTokenNumber++;
			};
	};

	std::tstringstream	strm(
		_T("[seperators]\n")
		_T("-\n")
		_T("--\n")
		_T("[tokens]\n")
		_T("test\n")
		_T("[rules]\n")
		_T("strings\n")
		_T("numbers\n") );

	cooLexxerMap				map;
	cooLexxerTextInputStream	tis(_T("test---test1--1234-\"test\""));
	cooTestListener				lis;
	cooLexxer					lexxer(&tis,&map,&lis);

	lis.nTokenNumber	=0;
	lis.fValid			=true;

	// Initialize map
	map.vLoadFromStream(strm);

	while(!tis.fIsEofReached())
		lexxer.vParseCharacter();

	return lis.fValid?lexxer.fCheckValid():false;
	}
#endif