Click here to Skip to main content
15,886,868 members
Articles / Programming Languages / C++

Tokenizer and analyzer package supporting precedence prioritized rules

Rate me:
Please Sign up or sign in to vote.
5.00/5 (4 votes)
1 Jan 20023 min read 181.7K   2.8K   54  
A library allowing you to conveniently build a custom tokenizer and analyzer supporting precedence priorized rules
// DlgEvaluate.cpp : Implementierungsdatei
//

#include "stdafx.h"
#include "grammarIDE.h"
#include "ltrItemTreeWnd.h"
#include "DlgEvaluate.h"


// CDlgEvaluate-Dialogfeld

IMPLEMENT_DYNAMIC(CDlgEvaluate, CDialog)
CDlgEvaluate::CDlgEvaluate(cxtPackage& pkg, CWnd* pParent /*=NULL*/)
: CDialog(CDlgEvaluate::IDD, pParent),
  m_pkg(pkg),
  m_treeWnd(&pkg)
	{
	m_papbRoot = NULL;
	}

CDlgEvaluate::~CDlgEvaluate()
	{
	delete m_papbRoot;
	}

void CDlgEvaluate::DoDataExchange(CDataExchange* pDX)
	{
	CDialog::DoDataExchange(pDX);
	DDX_Text(pDX,IDC_EVAL_SOURCE,m_strEvalSource);
	DDX_Text(pDX,IDC_TOKENIZING_MSEC,m_strMsecTokenizing);
	DDX_Text(pDX,IDC_ANALYZING_MSEC,m_strMsecAnalyzing);
	DDX_Control(pDX, IDC_RULE_TO_EVAL, m_ctlRuleCombo);
	}


BEGIN_MESSAGE_MAP(CDlgEvaluate, CDialog)
	ON_BN_CLICKED(IDC_EVALUATE, OnEvaluate)
	ON_BN_CLICKED(IDC_REBALANCE, OnRebalance)
	ON_COMMAND(ID_LTRCONTEXT_COLLAPSE, OnCollapse)
END_MESSAGE_MAP()

void CDlgEvaluate::OnCollapse()
	{
	AfxMessageBox("TRUE!");
	}


// CDlgEvaluate-Meldungshandler

BOOL CDlgEvaluate::OnInitDialog()
	{
	const cxAnalyzerTypeMap& atm = *m_pkg.patmGetAnalyzerMap();

	if(!CDialog::OnInitDialog())
		return FALSE;

	// Insert the rules elements into the rule select combo
	for(int i=1;;i++)
		{
		const cxAnalyzerTypeInfo *patiCur = atm.patiGetTypeInfo(i);
		if(patiCur==NULL)
			break;

		if(patiCur->nGetMType()!=ATM_MTYPE_RULE)
			continue;

		CString strRule;
		strRule = atm.strGetStringForID(i).c_str();
		int idx = m_ctlRuleCombo.AddString(strRule);
		m_ctlRuleCombo.SetItemData(idx,i);
		}
	m_ctlRuleCombo.SetCurSel(0);

	// Create the parse tree window
	CRect	rcTemp;
	GetDlgItem(IDC_TREEWND_BORDER)->GetClientRect(&rcTemp);
	GetDlgItem(IDC_TREEWND_BORDER)->MapWindowPoints(this,&rcTemp);
	m_treeWnd.Create(NULL,NULL,WS_CHILD|WS_VISIBLE,rcTemp,this,1000);

	return TRUE;
	}

cltrItemBranch *CDlgEvaluate::CreateTree(const cxaParseBranch *papbSource)
	{
	if( ((CGrammarIDEApp*)AfxGetApp())->isPreviewDisabled() )
		return NULL;

	const cxAnalyzerTypeMap& atm = *m_pkg.patmGetAnalyzerMap();
	int				nAtmType = papbSource->nGetAtmType();
	std::tstring	text = atm.strGetStringForID(nAtmType);
	cltrItemBranch	*pltrSource = new cltrItemBranch(text, papbSource);
	const cxaParseElement *papeCur = NULL;
	sit_data_t		sit;
	bool			fContinue = true;

	pltrSource->fIsExpanded=true;
	papbSource->vEnumBegin(&sit);
	while(fContinue)
		{
		papeCur		=papbSource->papeEnumGetAt(&sit);
		fContinue	=papbSource->fEnumNext(&sit);
		if(papeCur->fIsNull())
			continue;

		if(papeCur->fIsNode())
			{
			LPCTSTR lpszText = papeCur->lpszGetElemText();
			if(lpszText==NULL) lpszText="<NULL>";
			pltrSource->push_back(new cltrItemNode(lpszText,papeCur));
			}
		else
			pltrSource->push_back(CreateTree(papeCur->papbElem()));
		}
	papbSource->vEnumEnd(&sit);

	return pltrSource;
	}

void CDlgEvaluate::OnEvaluate()
	{
	delete m_papbRoot; m_papbRoot = NULL;
	int nAtmType = ATM_ID_INVALID;
	UpdateData(TRUE);

	// get the selected rule to test for
	if(m_ctlRuleCombo.GetCurSel()==CB_ERR)
		{
		AfxMessageBox("No rule selected.");
		return;
		}
	else
		nAtmType = m_ctlRuleCombo.GetItemData(m_ctlRuleCombo.GetCurSel());


	std::tstringstream	*pstrm = new std::tstringstream((LPCTSTR)m_strEvalSource);
	cxTokenizerSTLInputStream< std::tstringstream,cxTokenizerInputStreamDefaultWithLineNumberImpl<crlf_lfonly> >
		istream(pstrm,false);

		{
		LARGE_INTEGER freq,c0,c1,c2,c3,c4,d0,d1,d2;
		QueryPerformanceFrequency(&freq);
		QueryPerformanceCounter(&c0);
		QueryPerformanceCounter(&c1);

		// reset the tokenizer/analyzer context
		m_pkg.vReset();

		// attach the input stream to the tokenizer/analyzer
		m_pkg.vSetInputStream(&istream);
		m_pkg.vSetDelimeterIDs(NULL);
		m_pkg.nReadUntilDelimeter();
		QueryPerformanceCounter(&c2);

		// check for the given AtmType
		std::vector<std::tstring> ppErrors;
		cxaStatusCookie ascCondition;
		cxaTokenStream::const_iterator endpos;
		QueryPerformanceCounter(&c3);
		m_papbRoot		=m_pkg.papbCheckForRuleAtm(nAtmType,&endpos,&ascCondition,false);
		QueryPerformanceCounter(&c4);
		if(m_papbRoot==NULL)
			AfxMessageBox("Syntax error.");
	
		if(m_papbRoot!=NULL)
			{
			m_papbRoot->vDump();
			// create the tree structure for the window
			m_treeWnd.SetTree( CreateTree(m_papbRoot) );
			}

		if(m_pkg.fPreprocessorGetErrors(ppErrors))
			{
			std::tstring res;
			int i, size = ppErrors.size();
			for(i=0;i<size;i++)
				res += ppErrors[i];
			(size==0)?0:MessageBox(res.c_str(),"Preprocessor errors");
			}

		d0.QuadPart=c1.QuadPart-c0.QuadPart;
		d1.QuadPart=(c2.QuadPart-c1.QuadPart)-d0.QuadPart;
		d2.QuadPart=(c4.QuadPart-c3.QuadPart)-d0.QuadPart;
		double msecTokenizing = ((double)((d1.QuadPart*1000000L)/freq.QuadPart))/1000.0f;
		double msecAnalyzing = ((double)((d2.QuadPart*1000000L)/freq.QuadPart))/1000.0f;
		m_strMsecTokenizing.Format("%13.4f\n",msecTokenizing);
		m_strMsecAnalyzing.Format("%13.4f\n",msecAnalyzing);
		UpdateData(FALSE);

		// detach the input stream
		m_pkg.vSetInputStream(NULL);
		}
	}

void CDlgEvaluate::OnRebalance()
	{
	int nAtmType = ATM_ID_INVALID;
	UpdateData(TRUE);

	// get the selected rule to test for
	if(m_ctlRuleCombo.GetCurSel()==CB_ERR)
		{
		AfxMessageBox("No rule selected.");
		return;
		}
	else
		nAtmType = m_ctlRuleCombo.GetItemData(m_ctlRuleCombo.GetCurSel());

	if(m_papbRoot!=NULL)
		{
		m_pkg.vRebalanceAtm(m_papbRoot, nAtmType);

		// the first branch contains

		// create the tree structure for the window
		m_treeWnd.SetTree( CreateTree(m_papbRoot) );
		}
	}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here


Written By
Web Developer
Germany Germany
This member has not yet provided a Biography. Assume it's interesting and varied, and probably something to do with programming.

Comments and Discussions