Tokenizer and analyzer package supporting precedence prioritized rules

Alexander Berthold

Rate me:

5.00/5 (4 votes)

1 Jan 20023 min read

182K

2.8K

A library allowing you to conveniently build a custom tokenizer and analyzer supporting precedence priorized rules

pkgcomplete.zip
- COMMON.CPP
- COMMON.H
- common_error.h
- common_win32.h
- COPYING
- cpAbsd.dsw
- cpp-grammar.txt
- cxAnalyzer
  - ANALYZER.H
  - cxAnalyzer.dsp
  - cxAnalyzerException.h
  - cxAnalyzerExpression.cpp
  - cxAnalyzerExpression.h
  - cxAnalyzerMain.cpp
  - cxAnalyzerMain.h
  - cxAnalyzerTree.cpp
  - cxAnalyzerTree.h
  - cxAnalyzerTypeMap.cpp
  - cxAnalyzerTypeMap.h
  - cxaParseTree.cpp
  - cxaParseTree.h
  - cxaRuleCache.cpp
  - cxaRuleCache.h
  - cxaRuleCacheElement.cpp
  - cxaRuleCacheElement.h
  - cxaToken.cpp
  - cxaToken.h
  - cxaToken.inl
  - cxaTokenStream.cpp
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
- cxTokenizer
- cxtPackage
  - cxtPackage.cpp
  - cxtPackage.dsp
  - cxtPackage.h
  - PACKAGE.H
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
  - todo.txt
- DEBUG.CPP
- DEBUG.H
- emptyTestApp
  - emptyTestApp.clw
  - emptyTestApp.cpp
  - emptyTestApp.dsp
  - emptyTestApp.h
  - emptyTestApp.rc
  - emptyTestAppDlg.cpp
  - emptyTestAppDlg.h
  - ReadMe.txt
  - res
    - emptyTestApp.ico
    - emptyTestApp.rc2
  - Resource.h
  - StdAfx.cpp
  - StdAfx.h
- grammarIDE
  - DlgEvaluate.cpp
  - DlgEvaluate.h
  - DlgProperties.cpp
  - DlgProperties.h
  - grammarIDE.clw
  - grammarIDE.cpp
  - grammarIDE.dsp
  - grammarIDE.h
  - grammarIDE.rc
  - grammarIDEDoc.cpp
  - grammarIDEDoc.h
  - grammarIDEView.cpp
  - grammarIDEView.h
  - LeftView.cpp
  - LeftView.h
  - ltrItemTreeWnd.cpp
  - ltrItemTreeWnd.h
  - MainFrm.cpp
  - MainFrm.h
  - PaneEditorWnd.cpp
  - PaneEditorWnd.h
  - propsItemGrammar.cpp
  - propsItemGrammar.h
  - ReadMe.txt
  - res
    - grammarIDE.ico
    - grammarIDE.rc2
    - grammarIDEDoc.ico
    - icon1.ico
    - Toolbar.bmp
    - vssver.scc
    - zoomable.ico
  - Resource.h
  - StdAfx.cpp
  - StdAfx.h
- readme.txt
- sample-grammar.txt
- simpleCalc
  - ReadMe.txt
  - simpleCalc.bmp
  - simpleCalc.cpp
  - simpleCalc.dsp
  - StdAfx.cpp
  - StdAfx.h
- tkCommon
  - ctkCheckValid.h
  - ctkEnumerator.h
  - ctkExternalObjectPointer.h
  - ctkFlagsMixin.h
  - ctkHLinkedList.cpp
  - ctkHLinkedList.h
  - ctkMisc.h
  - ctkSerializable.h
  - tkCommon.h
  - vssver.scc
- vcstl_nowarnings.h
pkgsrconly.zip
- COMMON.CPP
- COMMON.H
- common_error.h
- common_win32.h
- COPYING
- cpAbsd.dsw
- cpp-grammar.txt
- ANALYZER.H
- cxAnalyzer.dsp
- cxAnalyzerException.h
- cxAnalyzerExpression.cpp
- cxAnalyzerExpression.h
- cxAnalyzerMain.cpp
- cxAnalyzerMain.h
- cxAnalyzerTree.cpp
- cxAnalyzerTree.h
- cxAnalyzerTypeMap.cpp
- cxAnalyzerTypeMap.h
- cxaParseTree.cpp
- cxaParseTree.h
- cxaRuleCache.cpp
- cxaRuleCache.h
- cxaRuleCacheElement.cpp
- cxaRuleCacheElement.h
- cxaToken.cpp
- cxaToken.h
- cxaToken.inl
- cxaTokenStream.cpp
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- cxTokenizer.cpp
- cxTokenizer.dsp
- cxTokenizer.h
- cxTokenizerCharTokenRule.cpp
- cxTokenizerCharTokenRule.h
- cxTokenizerCommentTokenRule.cpp
- cxTokenizerCommentTokenRule.h
- cxTokenizerContext.cpp
- cxTokenizerContext.h
- cxTokenizerContextCookie.h
- cxTokenizerContextDiags.cpp
- cxTokenizerDiags.cpp
- cxTokenizerException.cpp
- cxTokenizerException.h
- cxTokenizerInputStream.h
- cxTokenizerMap.cpp
- cxTokenizerMap.h
- cxTokenizerMapData.cpp
- cxTokenizerMapData.h
- cxTokenizerMapDataDiags.cpp
- cxTokenizerMapDiags.cpp
- cxTokenizerMatchTokenRule.h
- cxTokenizerMatchTokenRule.inl
- cxTokenizerNumberTokenRule.cpp
- cxTokenizerNumberTokenRule.h
- cxTokenizerSTLInputStream.cpp
- cxTokenizerSTLInputStream.h
- cxTokenizerStringTokenRule.cpp
- cxTokenizerStringTokenRule.h
- cxTokenizerTextInputStream.cpp
- cxTokenizerTextInputStream.h
- cxTokenizerTokenRule.cpp
- cxTokenizerTokenRule.h
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- tokenizer.h
- tokenizer_error.h
- tokenizer_flags.h
- tokenizerBase.h
- cxtPackage.cpp
- cxtPackage.dsp
- cxtPackage.h
- PACKAGE.H
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- todo.txt
- DEBUG.CPP
- DEBUG.H
- emptyTestApp.clw
- emptyTestApp.cpp
- emptyTestApp.dsp
- emptyTestApp.h
- emptyTestApp.rc
- emptyTestAppDlg.cpp
- emptyTestAppDlg.h
- ReadMe.txt
- emptyTestApp.ico
- emptyTestApp.rc2
- Resource.h
- StdAfx.cpp
- StdAfx.h
- readme.txt
- sample-grammar.txt
- ctkCheckValid.h
- ctkEnumerator.h
- ctkExternalObjectPointer.h
- ctkFlagsMixin.h
- ctkHLinkedList.cpp
- ctkHLinkedList.h
- ctkMisc.h
- ctkSerializable.h
- tkCommon.h
- vssver.scc
- vcstl_nowarnings.h
grammaride.zip
- cpp-grammar.txt
- grammarIDE.exe
- readme.txt
- sample-grammar.txt
- stlport_vc645.dll
cxtpackagetut_win32vc.zip
- common.cpp
- common.h
- common_error.h
- common_win32.h
- COPYING
- cpAbsd.dsw
- analyzer.h
- cxAnalyzer.dsp
- cxAnalyzer.plg
- cxAnalyzerException.h
- cxAnalyzerExpression.cpp
- cxAnalyzerExpression.h
- cxAnalyzerMain.cpp
- cxAnalyzerMain.h
- cxAnalyzerTree.cpp
- cxAnalyzerTree.h
- cxAnalyzerTypeMap.cpp
- cxAnalyzerTypeMap.h
- cxaParseTree.cpp
- cxaParseTree.h
- cxaToken.cpp
- cxaToken.h
- cxaTokenStream.cpp
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- cxTokenizer.cpp
- cxTokenizer.dsp
- cxTokenizer.h
- cxTokenizer.plg
- cxTokenizerCharTokenRule.cpp
- cxTokenizerCharTokenRule.h
- cxTokenizerContext.cpp
- cxTokenizerContext.h
- cxTokenizerContextCookie.h
- cxTokenizerContextDiags.cpp
- cxTokenizerDiags.cpp
- cxTokenizerException.cpp
- cxTokenizerException.h
- cxTokenizerInputStream.h
- cxTokenizerMap.cpp
- cxTokenizerMap.h
- cxTokenizerMapData.cpp
- cxTokenizerMapData.h
- cxTokenizerMapDataDiags.cpp
- cxTokenizerMapDiags.cpp
- cxTokenizerNumberTokenRule.cpp
- cxTokenizerNumberTokenRule.h
- cxTokenizerSTLInputStream.cpp
- cxTokenizerSTLInputStream.h
- cxTokenizerStringTokenRule.cpp
- cxTokenizerStringTokenRule.h
- cxTokenizerTextInputStream.cpp
- cxTokenizerTextInputStream.h
- cxTokenizerTokenRule.cpp
- cxTokenizerTokenRule.h
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- tokenizer.h
- tokenizer_error.h
- tokenizer_flags.h
- tokenizerBase.h
- cxtPackage.cpp
- cxtPackage.dsp
- cxtPackage.h
- cxtPackage.plg
- package.h
- Readme.txt
- StdAfx.cpp
- StdAfx.h
- mathTok
  - mathTok.cpp
  - mathTok.dsp
  - mathTok.plg
  - ReadMe.txt
  - StdAfx.cpp
  - StdAfx.h
  - ReadMe.txt
  - simpleCalc.bmp
  - simpleCalc.cpp
  - simpleCalc.dsp
  - simpleCalc.plg
  - StdAfx.cpp
  - StdAfx.h
  - ctkCheckValid.h
  - ctkExternalObjectPointer.h
  - ctkFlagsMixin.h
  - ctkHLinkedList.cpp
  - ctkHLinkedList.h
  - ctkMisc.h
  - ctkSerializable.h
  - tkCommon.h
cxtpackage_win32vc.zip
- common.cpp
- common.h
- common_error.h
- common_win32.h
- COPYING
- cpAbsd.dsw
- analyzer.h
- cxAnalyzer.dsp
- cxAnalyzer.plg
- cxAnalyzerException.h
- cxAnalyzerExpression.cpp
- cxAnalyzerExpression.h
- cxAnalyzerMain.cpp
- cxAnalyzerMain.h
- cxAnalyzerTree.cpp
- cxAnalyzerTree.h
- cxAnalyzerTypeMap.cpp
- cxAnalyzerTypeMap.h
- cxaParseTree.cpp
- cxaParseTree.h
- cxaToken.cpp
- cxaToken.h
- cxaTokenStream.cpp
- Debug
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
  - cxTokenizer.cpp
  - cxTokenizer.dsp
  - cxTokenizer.h
  - cxTokenizer.plg
  - cxTokenizerCharTokenRule.cpp
  - cxTokenizerCharTokenRule.h
  - cxTokenizerContext.cpp
  - cxTokenizerContext.h
  - cxTokenizerContextCookie.h
  - cxTokenizerContextDiags.cpp
  - cxTokenizerDiags.cpp
  - cxTokenizerException.cpp
  - cxTokenizerException.h
  - cxTokenizerInputStream.h
  - cxTokenizerMap.cpp
  - cxTokenizerMap.h
  - cxTokenizerMapData.cpp
  - cxTokenizerMapData.h
  - cxTokenizerMapDataDiags.cpp
  - cxTokenizerMapDiags.cpp
  - cxTokenizerNumberTokenRule.cpp
  - cxTokenizerNumberTokenRule.h
  - cxTokenizerSTLInputStream.cpp
  - cxTokenizerSTLInputStream.h
  - cxTokenizerStringTokenRule.cpp
  - cxTokenizerStringTokenRule.h
  - cxTokenizerTextInputStream.cpp
  - cxTokenizerTextInputStream.h
  - cxTokenizerTokenRule.cpp
  - cxTokenizerTokenRule.h
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
  - tokenizer.h
  - tokenizer_error.h
  - tokenizer_flags.h
  - tokenizerBase.h
  - cxtPackage.cpp
  - cxtPackage.dsp
  - cxtPackage.h
  - cxtPackage.plg
  - package.h
  - Readme.txt
  - StdAfx.cpp
  - StdAfx.h
  - ctkCheckValid.h
  - ctkExternalObjectPointer.h
  - ctkFlagsMixin.h
  - ctkHLinkedList.cpp
  - ctkHLinkedList.h
  - ctkMisc.h
  - ctkSerializable.h
  - tkCommon.h

// stdafx.h : include file for standard system include files,
//  or project specific include files that are used frequently, but
//      are changed infrequently
//

#if !defined(AFX_STDAFX_H__02256370_CAF8_42F4_A5C0_87765470A24F__INCLUDED_)
#define AFX_STDAFX_H__02256370_CAF8_42F4_A5C0_87765470A24F__INCLUDED_

#if _MSC_VER > 1000
#pragma once
#endif // _MSC_VER > 1000

#define VC_EXTRALEAN		// Exclude rarely-used stuff from Windows headers

#include <afx.h>
#include <afxwin.h>

// TODO: reference additional headers your program requires here
#include "../common.h"
#include "../cxTokenizer/tokenizer.h"


//{{AFX_INSERT_LOCATION}}
// Microsoft Visual C++ will insert additional declarations immediately before the previous line.

#endif // !defined(AFX_STDAFX_H__02256370_CAF8_42F4_A5C0_87765470A24F__INCLUDED_)

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

Written By

Alexander Berthold

Web Developer

Germany

This member has not yet provided a Biography. Assume it's interesting and varied, and probably something to do with programming.

Tokenizer and analyzer package supporting precedence prioritized rules

License

Comments and Discussions