Implementing a std::map Replacement that Never Runs Out of Memory and Instructions on Producing an ARPA Compliant Language Model to Test the Implementation

#ifndef __TOKENIZER_H__
#define __TOKENIZER_H__

#include <istream>

using namespace std;

// CTokenizer is a base class defining how tokenization should proceed.

class CTokenizer
{

public:

	// Constructor:

	// REQUIREMENTS:
	// An istream successfully opened for reading.
	// PROMISES:
	// The object will be ready to return tokens with GetNextToken if HasMoreToken return true.

	CTokenizer(istream &inputStream) throw();

	// Destructor:

	// REQUIREMENTS:
	// None.
	// PROMISES:
	// None.

	virtual ~CTokenizer() throw();

	// GetNextToken():

	// REQUIREMENTS:
	// HasMoreToken() must have returned true for this call to return an actual token, otherwise, it returns an empty string.
	// PROMISES:
	// The next token from the input stream.

	virtual string GetNextToken() throw()  = 0;

	// HasMoreToken():

	// REQUIREMENTS:
	// None.
	// PROMISES:
	// If the return value is true, GetNextToken() will return a token, otherwise, no more tokens are available from the input stream.

	virtual bool HasMoreToken() throw() = 0;

protected:

	istream &m_stream;
};

#endif