Tokenizer and analyzer package supporting precedence prioritized rules

/*********************************************************************
	Copyright (C) 2001 by

		Alexander Berthold, alexander-berthold@web.de.
		Hoegestr. 54
		79108 Freiburg i. Breisgau
		Germany

    -- This file is part of cxTokenizer --

    "cxTokenizer" is free software; you can redistribute it and/or 
	modify it under the terms of the GNU Lesser General Public 
	License as published by the Free Software Foundation; either 
	version 2 of the License, or any later version.

    "cxTokenizer" is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
	License along with "cxTokenizer"; if not, write to the Free 
	Software  Foundation, Inc., 59 Temple Place, Suite 330, 
	Boston, MA  02111-1307  USA

    ---------------------------------------------------------------
      If you find any bugs or if you make other corrections/
	  enhancements, i'd appreciate if you'd let me know about 
	  that. My email is
  
       alexander-berthold@web.de
  
      If you share this code, do not remove this text.
    ---------------------------------------------------------------

*********************************************************************/

// cxTokenizerContext.cpp: implementation of the cxTokenizerContext class.
//
//////////////////////////////////////////////////////////////////////

#include "stdafx.h"

#include "cxTokenizerMapData.h"
#include "cxTokenizerTokenRule.h"
#include "cxTokenizerContextCookie.h"
#include "cxTokenizerContext.h"
#include "cxTokenizer.h"
#include "cxTokenizerInputStream.h"

#ifdef _DEBUG
#undef THIS_FILE
static char THIS_FILE[]=__FILE__;
#define new DEBUG_NEW
#endif

#include "cxTokenizerContextDiags.cpp"

//////////////////////////////////////////////////////////////////////
// Construction/Destruction
//////////////////////////////////////////////////////////////////////

    cxTokenizerContext::cxTokenizerContext()
	    {
	    ctkFlagsMixin<xtctx_flags>::vClearAllFlags();
		m_pxtTokenizer=NULL;
	    }

    cxTokenizerContext::~cxTokenizerContext()
	    {
		vReset();
	    }

	void cxTokenizerContext::vReset()
		{
		tc_cookiemap_type::iterator it;
		for(it=m_mapCookies.begin();it!=m_mapCookies.end();it++)
			delete it->second;
		for(it=m_mapGlobalCookies.begin();it!=m_mapGlobalCookies.end();it++)
			delete it->second;
	    m_mapCookies.clear();
		m_mapGlobalCookies.clear();
	    m_lstTokenRules.clear();
		}

//////////////////////////////////////////////////////////////////////
// Cookie operations
//////////////////////////////////////////////////////////////////////

    /*********************************************************************
	    FUNCTION:	cxTokenizerContext::ptccGetCookie
	    PURPOSE:	Returns the cookie for the ID 'dwCookieID'
	    RETURNS:	cxTokenizerContextCookie*
    *********************************************************************/
    cxTokenizerContextCookie
		    *cxTokenizerContext::ptccGetCookie(const tc_cookiemap_type& map, DWORD dwCookieID) const
	    {
	    ASSERT(dwCookieID!=0);

	    const_cookiemap_iterator	it;
	    it					=map.find(dwCookieID);
	    if(it==map.end() || it->second->fIsDeleted())
		    return NULL;

	    ASSERT(it->second->fCheckValid());

	    return it->second;
	    }

    /*********************************************************************
	    FUNCTION:	cxTokenizerContext::fDeleteCookie
	    PURPOSE:	Deletes the cookie for the cookie ID 'dwCookieID'
	    RETURNS:	'true' if deleted
                    'false' if cookie did not exist
    *********************************************************************/
    bool	cxTokenizerContext::fDeleteCookie(DWORD dwCookieID)
	    {
	    ASSERT(dwCookieID!=0);

	    cookiemap_iterator	it;
	    it					=m_mapCookies.find(dwCookieID);
	    if(it==m_mapCookies.end())
		    return false;

		it->second->~cxTokenizerContextCookie();
		it->second->m_fDeleted = true;

	    return true;
	    }

    /*********************************************************************
	    FUNCTION:	cxTokenizerContext::fDeleteCookie
	    PURPOSE:	Deletes the cookie for the rule 'pltrRule'
	    RETURNS:	'true' if deleted
                    'false' if cookie did not exist
    *********************************************************************/
    bool	cxTokenizerContext::fDeleteCookie(cxTokenizerTokenRule *pttrRule)
	    {
	    ASSERT(pttrRule!=NULL);
	    ASSERT(pttrRule->fCheckValid());

	    // TODO: find better solution
	    DWORD	dwCookieID = (DWORD)pttrRule;

	    return fDeleteCookie(dwCookieID);
	    }

    /*********************************************************************
	    FUNCTION:	cxTokenizerContext::fDeleteCookie
	    PURPOSE:	Deletes the cookie for the cookie ID 'dwCookieID'
	    RETURNS:	'true' if deleted
                    'false' if cookie did not exist
    *********************************************************************/
    bool	cxTokenizerContext::fDeleteGlobalCookie(DWORD dwCookieID)
	    {
	    ASSERT(dwCookieID!=0);

	    cookiemap_iterator	it;
	    it					=m_mapGlobalCookies.find(dwCookieID);
	    if(it==m_mapGlobalCookies.end())
		    return false;

		delete (*it).second;
		m_mapGlobalCookies.erase(it);

	    return true;
	    }

    /*********************************************************************
	    FUNCTION:	cxTokenizerContext::fDeleteCookie
	    PURPOSE:	Deletes the cookie for the rule 'pltrRule'
	    RETURNS:	'true' if deleted
                    'false' if cookie did not exist
    *********************************************************************/
    bool	cxTokenizerContext::fDeleteGlobalCookie(cxTokenizerTokenRule *pttrRule)
	    {
	    ASSERT(pttrRule!=NULL);
	    ASSERT(pttrRule->fCheckValid());

	    // TODO: find better solution
	    DWORD	dwCookieID = (DWORD)pttrRule;

	    return fDeleteGlobalCookie(dwCookieID);
	    }

//////////////////////////////////////////////////////////////////////
// Data access operations
//////////////////////////////////////////////////////////////////////

	namespace utility
		{
		class	rule_is_deleted : public std::unary_function<cxTokenizerContext::cxListEntry*, bool>
			{
			public:
				bool operator() (const cxTokenizerContext::cxListEntry*& pEntry) const
					{
					return pEntry->fIsMarkedForDeletion();
					}
			};
		}

    /*********************************************************************
	    FUNCTION:	cxTokenizerContext::nDeleteMarkedListEntries
	    PURPOSE:	Removes all rules with 'fMarkedForDeletion' - flag
                    set from the list of the currently active rules
	    RETURNS:	int; count of removed rules
    *********************************************************************/
    int		cxTokenizerContext::nDeleteMarkedListEntries()
	    {
		ASSERT(fCheckValid());
		m_lstTokenRules.remove_if( utility::rule_is_deleted() );
		return 1;
	    }

    /*********************************************************************
	    FUNCTION:	cxTokenizerContext::vIncludeStream
	    PURPOSE:	"Pushes" a input stream which will be processed
					until EOS is reached, then input goes on with
					previous input stream
	    RETURNS:	- void -
    *********************************************************************/
    void	cxTokenizerContext::vIncludeStream(cxTokenizerInputStream *ptisInclude)
	    {
		ASSERT(fCheckValid());
		ASSERT(ptisInclude!=NULL);
		ASSERT(ptisInclude->fCheckValid());
		ASSERT(m_pxtTokenizer->fCheckValid());
		m_pxtTokenizer->vPushInputStream(ptisInclude);
	    }

    /*********************************************************************
	    FUNCTION:	cxTokenizerContext::vCleanUpAfterTokenRecognition
	    PURPOSE:	Is being called after a (delimeting!) token has 
                    been recognized (cxTokenizer::vDelimTokenRecognized).
                    Resets the lexxer context.
	    RETURNS:	- void -
    *********************************************************************/
    void	cxTokenizerContext::vCleanUpAfterTokenRecognition()
	    {
	    // delete cookies
	    cookiemap_iterator	cit;
	    for(cit=m_mapCookies.begin();cit!=m_mapCookies.end();cit++)
			{
			if(!cit->second->fIsDeleted())
				{
				cit->second->~cxTokenizerContextCookie();
				cit->second->m_fDeleted = true;
				}
			}

	    // delete token list
	    rulelist_iterator	it;
	    for(it=m_lstTokenRules.begin();it!=m_lstTokenRules.end();it++)
		    (*it)->vMarkForDeletion();
	    nDeleteMarkedListEntries();

	    // delete current text
	    m_strCurrentText	=_T("");
	    }

	TCHAR	cxTokenizerContext::tcPeekNextCharacter() const
		{
		ASSERT(m_pxtTokenizer!=NULL);
		ASSERT(m_pxtTokenizer->ptiGetInputStream()!=NULL);
		return m_pxtTokenizer->ptiGetInputStream()->tcPeekNextCharacter();
		}

	bool	cxTokenizerContext::fIsLastCharacter() const
		{
		ASSERT(m_pxtTokenizer!=NULL);
		ASSERT(m_pxtTokenizer->ptiGetInputStream()!=NULL);
		return m_pxtTokenizer->ptiGetInputStream()->fIsEofReached();
		}