Tokenizer and analyzer package supporting precedence prioritized rules

/*********************************************************************
	Copyright (C) 2001 by

		Alexander Berthold, alexander-berthold@web.de.
		Hoegestr. 54
		79108 Freiburg i. Breisgau
		Germany

    -- This file is part of cxTokenizer --

    "cxTokenizer" is free software; you can redistribute it and/or 
	modify it under the terms of the GNU Lesser General Public 
	License as published by the Free Software Foundation; either 
	version 2 of the License, or any later version.

    "cxTokenizer" is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
	License along with "cxTokenizer"; if not, write to the Free 
	Software  Foundation, Inc., 59 Temple Place, Suite 330, 
	Boston, MA  02111-1307  USA

    ---------------------------------------------------------------
      If you find any bugs or if you make other corrections/
	  enhancements, i'd appreciate if you'd let me know about 
	  that. My email is
  
       alexander-berthold@web.de
  
      If you share this code, do not remove this text.
    ---------------------------------------------------------------

*********************************************************************/

// cxTokenizerMatchTokenRule.inl: implementation of the cxTokenizerMatchTokenRule class.
//
//////////////////////////////////////////////////////////////////////




//////////////////////////////////////////////////////////////////////
// Construction/Destruction
//////////////////////////////////////////////////////////////////////

template<name_functor* pfunctor, int nHybridLength, TCHAR tcEnd, TCHAR tcEnd1, bool fCollect, bool fAcceptLastChar>
cxTokenizerMatchTokenRule<pfunctor,nHybridLength,tcEnd,tcEnd1,fCollect,fAcceptLastChar>
::cxTokenizerMatchTokenRule(int nIDValue, xttr_flags ttrf, const std::tstring& strInitString)
:	cxTokenizerTokenRule(nIDValue, ttrf,NULL,strInitString)
	{}

template<name_functor* pfunctor, int nHybridLength, TCHAR tcEnd, TCHAR tcEnd1, bool fCollect, bool fAcceptLastChar>
cxTokenizerMatchTokenRule<pfunctor,nHybridLength,tcEnd,tcEnd1,fCollect,fAcceptLastChar>
::~cxTokenizerMatchTokenRule()
	{}

//////////////////////////////////////////////////////////////////////
// Operations
//////////////////////////////////////////////////////////////////////

template<name_functor* pfunctor, int nHybridLength, TCHAR tcEnd, TCHAR tcEnd1, bool fCollect, bool fAcceptLastChar>
bool cxTokenizerMatchTokenRule<pfunctor,nHybridLength,tcEnd,tcEnd1,fCollect,fAcceptLastChar>
::fDoesApply(
	const cxTokenizerContext *ptcContext, TCHAR tcChar, bool *pfComplete, 
	int nStartPos, int nCurPos) const
	{
	const cxTokenizerMatchContextCookie
						*pCookie = NULL;
	ptcContext->fGetConstCookie(this,&pCookie);

	// +2: '/*'-Prefix
	if(nCurPos<(nStartPos+nHybridLength))
		return false;

	if(nCurPos==(nStartPos+nHybridLength))
		{
		if(pCookie!=NULL)
			return false;

		return true;
		}

	ASSERT(pCookie!=NULL);

	if(pCookie->fCompleted)
		return false;

	if(tcEnd1=='\0')
		{
		if(tcChar==tcEnd)
			{
			(*pfComplete)=true;
			return true;
			}
		}
	else
		{
		if(tcChar==tcEnd1 && pCookie->fEndExpected)
			{
			(*pfComplete)=true;
			return true;
			}
		}
	if(ptcContext->fIsLastCharacter())
		{
		if(fAcceptLastChar)
			(*pfComplete) = true;
		else
			return false;
		}

	return true;
	};

template<name_functor* pfunctor, int nHybridLength, TCHAR tcEnd, TCHAR tcEnd1, bool fCollect, bool fAcceptLastChar>
void cxTokenizerMatchTokenRule<pfunctor,nHybridLength,tcEnd,tcEnd1,fCollect,fAcceptLastChar>
::vApplied(
	cxTokenizerContext *ptcContext, TCHAR tcChar, bool fComplete, 
	int nStartPos, int nCurPos) const
	{
	cxTokenizerMatchContextCookie
						*pCookie = NULL;
	ptcContext->fGetCookie(this,&pCookie);

	// First character?
	if(nCurPos==nStartPos+nHybridLength)
		{
		// We need exclusive access to the following characters:
		ptcContext->vSetFlag(tctx_exclusive,true);

		ASSERT(pCookie==NULL);
		ptcContext->fSetCookie(this,&pCookie);
		return;
		}

	if( pCookie->fEndExpected || (tcEnd1=='\0' && tcChar==tcEnd))
		{
		// Release exclusive access
		ptcContext->vSetFlag(tctx_exclusive,false);

		// Set completed flag
		pCookie->fCompleted=true;
		return;
		}

	if( tcChar==tcEnd && tcEnd1!='\0' )
		{
		TCHAR	tcNext = ptcContext->tcPeekNextCharacter();
		if(tcNext==tcEnd1)
			pCookie->fEndExpected=true;
		}

	if(!pCookie->fEndExpected)
		pCookie->vAddToResultString(tcChar);
	};

template<name_functor* pfunctor, int nHybridLength, TCHAR tcEnd, TCHAR tcEnd1, bool fCollect, bool fAcceptLastChar>
bool	cxTokenizerMatchTokenRule<pfunctor,nHybridLength,tcEnd,tcEnd1,fCollect,fAcceptLastChar>
		::fGetResultString(const cxTokenizerContext *ptcContext, std::tstring& strResult) const
	{
	const cxTokenizerMatchContextCookie
						*pCookie = NULL;

	ptcContext->fGetCookie(this,&pCookie);
	ASSERT(pCookie!=NULL);

	strResult	=pCookie->strResult;
	return true;
	};

template<name_functor* pfunctor, int nHybridLength, TCHAR tcEnd, TCHAR tcEnd1, bool fCollect, bool fAcceptLastChar>
void	cxTokenizerMatchTokenRule<pfunctor,nHybridLength,tcEnd,tcEnd1,fCollect,fAcceptLastChar>
		::vSetArguments(const std::tstring& strArgs)
	{
	}