Tokenizer and analyzer package supporting precedence prioritized rules

/*********************************************************************
	Copyright (C) 2001/2 by

		Alexander Berthold, alexander-berthold@web.de.
		Hoegestr. 54
		79108 Freiburg i. Breisgau
		Germany

    -- This file is part of cxTokenizer --

    "cxTokenizer" is free software; you can redistribute it and/or 
	modify it under the terms of the GNU Lesser General Public 
	License as published by the Free Software Foundation; either 
	version 2 of the License, or any later version.

    "cxTokenizer" is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
	License along with "cxTokenizer"; if not, write to the Free 
	Software  Foundation, Inc., 59 Temple Place, Suite 330, 
	Boston, MA  02111-1307  USA

    ---------------------------------------------------------------
      If you find any bugs or if you make other corrections/
	  enhancements, i'd appreciate if you'd let me know about 
	  that. My email is
  
       alexander-berthold@web.de
  
      If you share this code, do not remove this text.
    ---------------------------------------------------------------

Class:      cxTokenizerTokenRule
Author:     Alexander Berthold
Copyright:  Alexander Berthold
Date:       2001/12/19
Version:	0.2.01
Purpose:    Base class for rules for the lexical analyzer 'tokenizer'


Version history:

	-	2001/05/19
		Renamed class from 'cpLexxerTokenRule' to 'cxTokenizerTokenRule'.

	-	2001/12/19
		Current source labeled version 0.2.01
		
*********************************************************************/

// cxTokenizerTokenRule.h: interface for the cxTokenizerTokenRule class.
//
//////////////////////////////////////////////////////////////////////

#if !defined(AFX_CXTOKENIZERTOKENRULE_H__CE5A4B37_A2E3_44BD_BFB7_099D65CD9B7F__INCLUDED_)
#define AFX_CXTOKENIZERTOKENRULE_H__CE5A4B37_A2E3_44BD_BFB7_099D65CD9B7F__INCLUDED_

#if _MSC_VER > 1000
#pragma once
#endif // _MSC_VER > 1000

#pragma warning(disable: 4100 4127)

class cxTokenizerContext;
class cxTokenizer;
class cxTokenizerInputStream;

class cxTokenizerSubstitutionListener
{
// Operations
public:
	virtual	bool	fDoSubstituteUnknownToken(
						cxTokenizerContext *ptcContext,	
						const std::tstring& strToken,
						const std::tstring& strDelim,
						cxTokenizerInputStream* ptisStream,
						std::tstring& strSubstitute) PURE_VIRTUAL_FALSE;
};

class cxTokenizerTokenRule : 
	public ctkFlagsMixin<xttr_flags>,
	public ctkCheckValid
{
// Construction/Destruction
protected:
	// lpszToken==NULL: Not a 'character' - Rule
	// lpszToken!=NULL: Character - Rule
	cxTokenizerTokenRule(int nIDValue, xttr_flags ltrf, LPCTSTR lpszToken, const std::tstring& strInitString);

public: 
	virtual ~cxTokenizerTokenRule();

// Attributes
protected:
	std::tstring		m_strInitString;
	std::tstring		m_strTokenString;
    int                 m_nIDValue;

// Operations
public:
    int                 nGetIDValue() const
                        { return m_nIDValue; };

	virtual	bool		fCheckValid() const
						{ return true; };
	virtual void		vSetArguments(const std::tstring& /*strArgs*/)
						{};
	virtual bool		fWantPutBack(const cxTokenizerContext *ptcContext) const
						{ return false; };
	virtual void		vGetPutBackString(const cxTokenizerContext *ptcContext, std::tstring& /*strPutback*/) const
						{};
	virtual void		vInitNotify(bool fInit, cxTokenizer *pxTokenizer)
						{};

	virtual std::tstring strGetTokenString() const;
	std::tstring		strGetInitString() const { return m_strInitString; };
	void				vSetInitString(const std::tstring& str) { m_strInitString = str; };

	virtual bool		fDoesApply(	const cxTokenizerContext *ptcContext, TCHAR tcChar, bool *pfComplete, 
									int nStartPos, int nCurPos) const PURE_VIRTUAL_FALSE;
	virtual void		vApplied( cxTokenizerContext *ptcContext, TCHAR tcChar, bool fComplete, 
								  int nStartPos, int nCurPos) const PURE_VIRTUAL;
	virtual bool		fGetResultString(const cxTokenizerContext *ptcContext, std::tstring& strResult) const PURE_VIRTUAL_FALSE;
};

class cxDummyTokenizerTokenRule : public cxTokenizerTokenRule
{
public:
	cxDummyTokenizerTokenRule(int nIDValue, LPCTSTR lpszToken)
		: cxTokenizerTokenRule(nIDValue,ttrf_dummy,lpszToken,"") {};

public:
	virtual std::tstring strGetTokenString() const { return m_strTokenString; };
	virtual bool		fDoesApply(	const cxTokenizerContext *ptcContext, TCHAR tcChar, bool *pfComplete, 
									int nStartPos, int nCurPos) const { return false; };
	virtual void		vApplied( cxTokenizerContext *ptcContext, TCHAR tcChar, bool fComplete, 
								  int nStartPos, int nCurPos) const { ASSERT(FALSE); };
	virtual bool		fGetResultString(const cxTokenizerContext *ptcContext, std::tstring& strResult) const 
								{ ASSERT(FALSE); return false; };;

};

#endif // !defined(AFX_CXTOKENIZERTOKENRULE_H__CE5A4B37_A2E3_44BD_BFB7_099D65CD9B7F__INCLUDED_)