Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Fast tokenizer

, 5 Sep 2001
Fast tokenizer for C++ - like 'lexx'
// cooLexxerTextInputStream.cpp: implementation of the cooLexxerTextInputStream class.
//
//////////////////////////////////////////////////////////////////////

#include "stdafx.h"

#include "cooLexxerInputStream.h"
#include "cooLexxerTextInputStream.h"

#ifdef _DEBUG
#undef THIS_FILE
static char THIS_FILE[]=__FILE__;
#define new DEBUG_NEW
#endif

//////////////////////////////////////////////////////////////////////
// Construction/Destruction
//////////////////////////////////////////////////////////////////////

cooLexxerTextInputStream::cooLexxerTextInputStream(LPCTSTR lpszInputData)
	{
	m_strInputData	=lpszInputData;
	m_strPutBackData=_T("");
	m_nInputPos		=0;
	m_nPutBackPos	=0;
	}

cooLexxerTextInputStream::~cooLexxerTextInputStream()
	{
	
	}

//////////////////////////////////////////////////////////////////////
// cooLexxerInputStream Operations
//////////////////////////////////////////////////////////////////////

bool	cooLexxerTextInputStream::fCheckValid() const
	{
	return true;
	}

TCHAR	cooLexxerTextInputStream::tcGetNextCharacter()
	{
	int		nInputLength = m_strInputData.length();
	int		nPutBackLength = m_strPutBackData.length();
	TCHAR	tcReturn = _T('\0');
	bool	fEof = fIsEofReached();

	ASSERT(!fEof);
	if(fEof)
		throw cooLexxerException(ERR_READ_PAST_EOF);

	if(m_nPutBackPos < nPutBackLength)
		{
		tcReturn	=m_strPutBackData[m_nPutBackPos++];
		if(m_nPutBackPos == nPutBackLength)
			{
			m_nPutBackPos	=0;
			m_strPutBackData=_T("");
			}
		}
	else
		{
		tcReturn	=m_strInputData[m_nInputPos++];
		}

	return tcReturn;
	}

bool	cooLexxerTextInputStream::fPutBack(LPCTSTR lpszText)
	{
	m_strPutBackData.insert(0,lpszText);
	return true;
	}

bool	cooLexxerTextInputStream::fPutBack(TCHAR tcChar)
	{
	m_strPutBackData.insert(0,tcChar);
	return true;
	}

bool	cooLexxerTextInputStream::fIsEofReached() const
	{
	int		nInputLength = m_strInputData.length();
	int		nPutBackLength = m_strPutBackData.length();

	if(	(m_nPutBackPos == nPutBackLength) &&
		(m_nInputPos == nInputLength))
		return true;

	return false;
	}

bool	cooLexxerTextInputStream::fShouldDelete() const
	{
	return false;
	}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

Share

About the Author

Alexander Berthold
Web Developer
Germany Germany
No Biography provided

| Advertise | Privacy | Mobile
Web04 | 2.8.140827.1 | Last Updated 6 Sep 2001
Article Copyright 2001 by Alexander Berthold
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid