Tokenizer and analyzer package supporting precedence prioritized rules

Alexander Berthold

Rate me:

5.00/5 (4 votes)

1 Jan 20023 min read

180.6K

2.8K

A library allowing you to conveniently build a custom tokenizer and analyzer supporting precedence priorized rules

pkgcomplete.zip
- COMMON.CPP
- COMMON.H
- common_error.h
- common_win32.h
- COPYING
- cpAbsd.dsw
- cpp-grammar.txt
- cxAnalyzer
  - ANALYZER.H
  - cxAnalyzer.dsp
  - cxAnalyzerException.h
  - cxAnalyzerExpression.cpp
  - cxAnalyzerExpression.h
  - cxAnalyzerMain.cpp
  - cxAnalyzerMain.h
  - cxAnalyzerTree.cpp
  - cxAnalyzerTree.h
  - cxAnalyzerTypeMap.cpp
  - cxAnalyzerTypeMap.h
  - cxaParseTree.cpp
  - cxaParseTree.h
  - cxaRuleCache.cpp
  - cxaRuleCache.h
  - cxaRuleCacheElement.cpp
  - cxaRuleCacheElement.h
  - cxaToken.cpp
  - cxaToken.h
  - cxaToken.inl
  - cxaTokenStream.cpp
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
- cxTokenizer
- cxtPackage
  - cxtPackage.cpp
  - cxtPackage.dsp
  - cxtPackage.h
  - PACKAGE.H
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
  - todo.txt
- DEBUG.CPP
- DEBUG.H
- emptyTestApp
  - emptyTestApp.clw
  - emptyTestApp.cpp
  - emptyTestApp.dsp
  - emptyTestApp.h
  - emptyTestApp.rc
  - emptyTestAppDlg.cpp
  - emptyTestAppDlg.h
  - ReadMe.txt
  - res
    - emptyTestApp.ico
    - emptyTestApp.rc2
  - Resource.h
  - StdAfx.cpp
  - StdAfx.h
- grammarIDE
  - DlgEvaluate.cpp
  - DlgEvaluate.h
  - DlgProperties.cpp
  - DlgProperties.h
  - grammarIDE.clw
  - grammarIDE.cpp
  - grammarIDE.dsp
  - grammarIDE.h
  - grammarIDE.rc
  - grammarIDEDoc.cpp
  - grammarIDEDoc.h
  - grammarIDEView.cpp
  - grammarIDEView.h
  - LeftView.cpp
  - LeftView.h
  - ltrItemTreeWnd.cpp
  - ltrItemTreeWnd.h
  - MainFrm.cpp
  - MainFrm.h
  - PaneEditorWnd.cpp
  - PaneEditorWnd.h
  - propsItemGrammar.cpp
  - propsItemGrammar.h
  - ReadMe.txt
  - res
    - grammarIDE.ico
    - grammarIDE.rc2
    - grammarIDEDoc.ico
    - icon1.ico
    - Toolbar.bmp
    - vssver.scc
    - zoomable.ico
  - Resource.h
  - StdAfx.cpp
  - StdAfx.h
- readme.txt
- sample-grammar.txt
- simpleCalc
  - ReadMe.txt
  - simpleCalc.bmp
  - simpleCalc.cpp
  - simpleCalc.dsp
  - StdAfx.cpp
  - StdAfx.h
- tkCommon
  - ctkCheckValid.h
  - ctkEnumerator.h
  - ctkExternalObjectPointer.h
  - ctkFlagsMixin.h
  - ctkHLinkedList.cpp
  - ctkHLinkedList.h
  - ctkMisc.h
  - ctkSerializable.h
  - tkCommon.h
  - vssver.scc
- vcstl_nowarnings.h
pkgsrconly.zip
- COMMON.CPP
- COMMON.H
- common_error.h
- common_win32.h
- COPYING
- cpAbsd.dsw
- cpp-grammar.txt
- ANALYZER.H
- cxAnalyzer.dsp
- cxAnalyzerException.h
- cxAnalyzerExpression.cpp
- cxAnalyzerExpression.h
- cxAnalyzerMain.cpp
- cxAnalyzerMain.h
- cxAnalyzerTree.cpp
- cxAnalyzerTree.h
- cxAnalyzerTypeMap.cpp
- cxAnalyzerTypeMap.h
- cxaParseTree.cpp
- cxaParseTree.h
- cxaRuleCache.cpp
- cxaRuleCache.h
- cxaRuleCacheElement.cpp
- cxaRuleCacheElement.h
- cxaToken.cpp
- cxaToken.h
- cxaToken.inl
- cxaTokenStream.cpp
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- cxTokenizer.cpp
- cxTokenizer.dsp
- cxTokenizer.h
- cxTokenizerCharTokenRule.cpp
- cxTokenizerCharTokenRule.h
- cxTokenizerCommentTokenRule.cpp
- cxTokenizerCommentTokenRule.h
- cxTokenizerContext.cpp
- cxTokenizerContext.h
- cxTokenizerContextCookie.h
- cxTokenizerContextDiags.cpp
- cxTokenizerDiags.cpp
- cxTokenizerException.cpp
- cxTokenizerException.h
- cxTokenizerInputStream.h
- cxTokenizerMap.cpp
- cxTokenizerMap.h
- cxTokenizerMapData.cpp
- cxTokenizerMapData.h
- cxTokenizerMapDataDiags.cpp
- cxTokenizerMapDiags.cpp
- cxTokenizerMatchTokenRule.h
- cxTokenizerMatchTokenRule.inl
- cxTokenizerNumberTokenRule.cpp
- cxTokenizerNumberTokenRule.h
- cxTokenizerSTLInputStream.cpp
- cxTokenizerSTLInputStream.h
- cxTokenizerStringTokenRule.cpp
- cxTokenizerStringTokenRule.h
- cxTokenizerTextInputStream.cpp
- cxTokenizerTextInputStream.h
- cxTokenizerTokenRule.cpp
- cxTokenizerTokenRule.h
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- tokenizer.h
- tokenizer_error.h
- tokenizer_flags.h
- tokenizerBase.h
- cxtPackage.cpp
- cxtPackage.dsp
- cxtPackage.h
- PACKAGE.H
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- todo.txt
- DEBUG.CPP
- DEBUG.H
- emptyTestApp.clw
- emptyTestApp.cpp
- emptyTestApp.dsp
- emptyTestApp.h
- emptyTestApp.rc
- emptyTestAppDlg.cpp
- emptyTestAppDlg.h
- ReadMe.txt
- emptyTestApp.ico
- emptyTestApp.rc2
- Resource.h
- StdAfx.cpp
- StdAfx.h
- readme.txt
- sample-grammar.txt
- ctkCheckValid.h
- ctkEnumerator.h
- ctkExternalObjectPointer.h
- ctkFlagsMixin.h
- ctkHLinkedList.cpp
- ctkHLinkedList.h
- ctkMisc.h
- ctkSerializable.h
- tkCommon.h
- vssver.scc
- vcstl_nowarnings.h
grammaride.zip
- cpp-grammar.txt
- grammarIDE.exe
- readme.txt
- sample-grammar.txt
- stlport_vc645.dll
cxtpackagetut_win32vc.zip
- common.cpp
- common.h
- common_error.h
- common_win32.h
- COPYING
- cpAbsd.dsw
- analyzer.h
- cxAnalyzer.dsp
- cxAnalyzer.plg
- cxAnalyzerException.h
- cxAnalyzerExpression.cpp
- cxAnalyzerExpression.h
- cxAnalyzerMain.cpp
- cxAnalyzerMain.h
- cxAnalyzerTree.cpp
- cxAnalyzerTree.h
- cxAnalyzerTypeMap.cpp
- cxAnalyzerTypeMap.h
- cxaParseTree.cpp
- cxaParseTree.h
- cxaToken.cpp
- cxaToken.h
- cxaTokenStream.cpp
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- cxTokenizer.cpp
- cxTokenizer.dsp
- cxTokenizer.h
- cxTokenizer.plg
- cxTokenizerCharTokenRule.cpp
- cxTokenizerCharTokenRule.h
- cxTokenizerContext.cpp
- cxTokenizerContext.h
- cxTokenizerContextCookie.h
- cxTokenizerContextDiags.cpp
- cxTokenizerDiags.cpp
- cxTokenizerException.cpp
- cxTokenizerException.h
- cxTokenizerInputStream.h
- cxTokenizerMap.cpp
- cxTokenizerMap.h
- cxTokenizerMapData.cpp
- cxTokenizerMapData.h
- cxTokenizerMapDataDiags.cpp
- cxTokenizerMapDiags.cpp
- cxTokenizerNumberTokenRule.cpp
- cxTokenizerNumberTokenRule.h
- cxTokenizerSTLInputStream.cpp
- cxTokenizerSTLInputStream.h
- cxTokenizerStringTokenRule.cpp
- cxTokenizerStringTokenRule.h
- cxTokenizerTextInputStream.cpp
- cxTokenizerTextInputStream.h
- cxTokenizerTokenRule.cpp
- cxTokenizerTokenRule.h
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- tokenizer.h
- tokenizer_error.h
- tokenizer_flags.h
- tokenizerBase.h
- cxtPackage.cpp
- cxtPackage.dsp
- cxtPackage.h
- cxtPackage.plg
- package.h
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- mathTok
  - mathTok.cpp
  - mathTok.dsp
  - mathTok.plg
  - ReadMe.txt
  - StdAfx.cpp
  - StdAfx.h
  - ReadMe.txt
  - simpleCalc.bmp
  - simpleCalc.cpp
  - simpleCalc.dsp
  - simpleCalc.plg
  - StdAfx.cpp
  - StdAfx.h
  - ctkCheckValid.h
  - ctkExternalObjectPointer.h
  - ctkFlagsMixin.h
  - ctkHLinkedList.cpp
  - ctkHLinkedList.h
  - ctkMisc.h
  - ctkSerializable.h
  - tkCommon.h
cxtpackage_win32vc.zip
- common.cpp
- common.h
- common_error.h
- common_win32.h
- COPYING
- cpAbsd.dsw
- analyzer.h
- cxAnalyzer.dsp
- cxAnalyzer.plg
- cxAnalyzerException.h
- cxAnalyzerExpression.cpp
- cxAnalyzerExpression.h
- cxAnalyzerMain.cpp
- cxAnalyzerMain.h
- cxAnalyzerTree.cpp
- cxAnalyzerTree.h
- cxAnalyzerTypeMap.cpp
- cxAnalyzerTypeMap.h
- cxaParseTree.cpp
- cxaParseTree.h
- cxaToken.cpp
- cxaToken.h
- cxaTokenStream.cpp
- Debug
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
  - cxTokenizer.cpp
  - cxTokenizer.dsp
  - cxTokenizer.h
  - cxTokenizer.plg
  - cxTokenizerCharTokenRule.cpp
  - cxTokenizerCharTokenRule.h
  - cxTokenizerContext.cpp
  - cxTokenizerContext.h
  - cxTokenizerContextCookie.h
  - cxTokenizerContextDiags.cpp
  - cxTokenizerDiags.cpp
  - cxTokenizerException.cpp
  - cxTokenizerException.h
  - cxTokenizerInputStream.h
  - cxTokenizerMap.cpp
  - cxTokenizerMap.h
  - cxTokenizerMapData.cpp
  - cxTokenizerMapData.h
  - cxTokenizerMapDataDiags.cpp
  - cxTokenizerMapDiags.cpp
  - cxTokenizerNumberTokenRule.cpp
  - cxTokenizerNumberTokenRule.h
  - cxTokenizerSTLInputStream.cpp
  - cxTokenizerSTLInputStream.h
  - cxTokenizerStringTokenRule.cpp
  - cxTokenizerStringTokenRule.h
  - cxTokenizerTextInputStream.cpp
  - cxTokenizerTextInputStream.h
  - cxTokenizerTokenRule.cpp
  - cxTokenizerTokenRule.h
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
  - tokenizer.h
  - tokenizer_error.h
  - tokenizer_flags.h
  - tokenizerBase.h
  - cxtPackage.cpp
  - cxtPackage.dsp
  - cxtPackage.h
  - cxtPackage.plg
  - package.h
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
  - ctkCheckValid.h
  - ctkExternalObjectPointer.h
  - ctkFlagsMixin.h
  - ctkHLinkedList.cpp
  - ctkHLinkedList.h
  - ctkMisc.h
  - ctkSerializable.h
  - tkCommon.h

// stdafx.h : include file for standard system include files,
//  or project specific include files that are used frequently, but
//      are changed infrequently
//

#if !defined(AFX_STDAFX_H__9C5BC0DD_EE6C_4EEC_A44B_585DD15E3B4C__INCLUDED_)
#define AFX_STDAFX_H__9C5BC0DD_EE6C_4EEC_A44B_585DD15E3B4C__INCLUDED_

#if _MSC_VER > 1000
#pragma once
#endif // _MSC_VER > 1000

#define VC_EXTRALEAN		// Exclude rarely-used stuff from Windows headers

#include <afx.h>
#include <afxwin.h>
#include "../common.h"
#include "tokenizerBase.h"

// TODO: reference additional headers your program requires here

//{{AFX_INSERT_LOCATION}}
// Microsoft Visual C++ will insert additional declarations immediately before the previous line.

#endif // !defined(AFX_STDAFX_H__9C5BC0DD_EE6C_4EEC_A44B_585DD15E3B4C__INCLUDED_)

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

Written By

Alexander Berthold

Web Developer

Germany

This member has not yet provided a Biography. Assume it's interesting and varied, and probably something to do with programming.

Tokenizer and analyzer package supporting precedence prioritized rules

License

Comments and Discussions