Tokenizer and analyzer package supporting precedence prioritized rules

/*********************************************************************
	Copyright (C) 2001 by

		Alexander Berthold, alexander-berthold@web.de.
		Hoegestr. 54
		79108 Freiburg i. Breisgau
		Germany

    -- This file is part of cxTokenizer --

    "cxTokenizer" is free software; you can redistribute it and/or 
	modify it under the terms of the GNU Lesser General Public 
	License as published by the Free Software Foundation; either 
	version 2 of the License, or any later version.

    "cxTokenizer" is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
	License along with "cxTokenizer"; if not, write to the Free 
	Software  Foundation, Inc., 59 Temple Place, Suite 330, 
	Boston, MA  02111-1307  USA

    ---------------------------------------------------------------
      If you find any bugs or if you make other corrections/
	  enhancements, i'd appreciate if you'd let me know about 
	  that. My email is
  
       alexander-berthold@web.de
  
      If you share this code, do not remove this text.
    ---------------------------------------------------------------

Class:      cxTokenizerSTLInputStream
Author:     Alexander Berthold
Copyright:  Alexander Berthold
Date:       2001/06/12
Version:	0.1.16
Purpose:    Sample input stream class for the parser.
            Takes a STL input stream as constructor argument and creates
            a cxTokenizerInputStream compatible stream out of it.


Version history:

	-	2001/05/19
		Renamed class from 'cpLexxerSTLInputStream' to 'cxTokenizerSTLInputStream'.

	-	2001/06/12
		Current source labeled version 0.1.16
		
*********************************************************************/

// cxTokenizerSTLInputStream.h: interface for the cxTokenizerSTLInputStream class.
//
//////////////////////////////////////////////////////////////////////

#if !defined(AFX_CXTOKENIZERSTLINPUTSTREAM_H__E5486C62_68E8_4E19_B3E7_534F2A57460F__INCLUDED_)
#define AFX_CXTOKENIZERSTLINPUTSTREAM_H__E5486C62_68E8_4E19_B3E7_534F2A57460F__INCLUDED_

#if _MSC_VER > 1000
#pragma once
#endif // _MSC_VER > 1000

template<class T, class TBase, bool fBinary = false, int eof_char = 26>
class cxTokenizerSTLInputStream : public TBase
{
// Construction/Destruction
public:
	cxTokenizerSTLInputStream(T& stlIStream, bool fShouldDelete = false)
	:	m_stlIStream(stlIStream)
		{
		m_strPutBackData=_T("");
		m_nPutBackPos	=0;

		m_tcNextChar	='\0';
		m_fEof			=false;
		m_fShouldDelete	=fShouldDelete;

		vGetNextChar();
		};

	virtual ~cxTokenizerSTLInputStream()
		{};

// Attributes
protected:
	T&				m_stlIStream;

	TCHAR			m_tcNextChar;
	bool			m_fEof;

	std::tstring	m_strInputData;
	int				m_nInputPos;
	std::tstring	m_strPutBackData;
	int				m_nPutBackPos;

	bool			m_fShouldDelete;

// Protected operations
protected:
	void			vGetNextChar()
		{
		ASSERT(m_fEof==false);
		T::int_type		tmp;

		tmp				=m_stlIStream.get();

		if(fBinary)
			{
			if(m_stlIStream.eof())
				m_fEof			=true;
			else
				m_tcNextChar	=(TCHAR)tmp;
			}
		else
			{
			if(m_stlIStream.eof() || tmp==std::char_traits<T::char_type>::eof())
				m_fEof			=true;
			else
				m_tcNextChar	=(TCHAR)tmp;
			}
		};

	virtual bool	fReadBlock(long lStartPos, long lLength, std::tstring& strResult, TCHAR tcTermChar) const
		{
		ASSERT(lLength>=0);
		ASSERT(lStartPos>=0);

		if(m_stlIStream.fail())
			return false;

		TCHAR	szBuffer = (TCHAR*)_alloca((lLength+1)*sizeof(TCHAR));
		T::pos_type cpos = m_stlIStream.tellg();
		m_stlIStream.seekg(lStartPos);
		strResult.reserve(lLength);
		m_stlIStream.get(szBuffer,lLength,tcTermChar);
		m_stlIStream.seekg(cpos);

		strResult = szBuffer;

		return true;
		};

// Operations
public:
	virtual	bool	fCheckValid() const
		{
		return true;
		};

	virtual TCHAR	tcGetNextCharacter()
		{
		bool	fEof = fIsEofReached();
		int		nPutBackLength = m_strPutBackData.length();
		TCHAR	tcReturn = _T('\0');

		ASSERT(!fEof);
		if(fEof)
			throw cxTokenizerException(ERR_READ_PAST_EOF);

		if(m_nPutBackPos < nPutBackLength)
			{
			tcReturn	=m_strPutBackData[m_nPutBackPos++];
			if(m_nPutBackPos == nPutBackLength)
				{
				m_nPutBackPos	=0;
				m_strPutBackData=_T("");
				}
			}
		else
			{
			tcReturn	=m_tcNextChar;
			vGetNextChar();
			}

		return tcGetNextCharacterDefaultImpl(tcReturn);
		};

	virtual bool	fPutBack(LPCTSTR lpszText)
		{
		m_strPutBackData.insert(0,lpszText);
		return fPutBackDefaultImpl(lpszText);
		};

	virtual bool	fPutBack(TCHAR tcChar)
		{
		m_strPutBackData.insert(0,tcChar);
		return fPutBackDefaultImpl(tcChar);
		};

	virtual bool	fIsEofReached() const
		{
		int		nPutBackLength = m_strPutBackData.length();

		if(m_nPutBackPos<nPutBackLength)
			return false;

		return m_fEof;
		};

	virtual bool	fShouldDelete() const
		{
		return m_fShouldDelete;
		};
};

#endif // !defined(AFX_CXTOKENIZERSTLINPUTSTREAM_H__E5486C62_68E8_4E19_B3E7_534F2A57460F__INCLUDED_)