Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Wave: a Standard conformant C++ preprocessor library

, 10 Jan 2004
Describes a free and fully Standard conformant C++ preprocessor library
wave_preprocessor_demo.zip
wave.exe
wave_preprocessor_demo1.zip
wave.exe
wave_preprocessor_src.zip
wave
doc
theme
bkd.gif
bkd2.gif
bullet.gif
l_arr.gif
l_arr_disabled.gif
r_arr.gif
r_arr_disabled.gif
uc.gif
u_arr.gif
wave.gif
test
boost-build.jam
cpp_tokens
Jamfile.v2
list_includes
project-root.jam
wave
Jamfile
Jamfile.v2
runtests.sh
test_files
wave
cpplexer
re2clex
cpp.re
test
lextest.in
lextest.output
run_tests.sh
test_lexer.in
test_lexer.output
slex
test
lextest.in
lextest.re2c.output
lextest.slex.output
run_tests.sh
grammars
idllexer
re2clex
idl.re
util
wave_preprocessor_src1.zip
bkd.gif
bkd2.gif
bullet.gif
l_arr.gif
l_arr_disabled.gif
r_arr.gif
r_arr_disabled.gif
u_arr.gif
uc.gif
wave.gif
boost-build.jam
Jamfile.v2
project-root.jam
Jamfile
Jamfile.v2
runtests.sh
cpp.re
lextest.in
lextest.output
run_tests.sh
test_lexer.in
test_lexer.output
lextest.in
lextest.re2c.output
lextest.slex.output
run_tests.sh
/*=============================================================================
    Wave: A Standard compliant C++ preprocessor

    Re2C based C++ lexer
    
    Copyright (c) 2001-2003 Hartmut Kaiser
    http://spirit.sourceforge.net/

    Permission to copy, use, modify, sell and distribute this software
    is granted provided this copyright notice appears in all copies.
    This software is provided "as is" without express or implied
    warranty, and with no claim as to its suitability for any purpose.

    See Copyright.txt for full copyright notices and acknowledgements.
=============================================================================*/

#if !defined(CPP_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED)
#define CPP_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED

#include <string>
#include <cstdio>
#include <cstdarg>
#if defined(BOOST_SPIRIT_DEBUG)
#include <iostream>
#endif // defined(BOOST_SPIRIT_DEBUG)

#include <boost/concept_check.hpp>
#include <boost/spirit/core.hpp>
#include <boost/spirit/core/assert.hpp>

#include "wave/util/file_position.hpp"
#include "wave/cpplexer/validate_universal_char.hpp"
#include "wave/cpplexer/cpplexer_exceptions.hpp"
#include "wave/cpplexer/cpp_token_ids.hpp"
#include "wave/cpplexer/cpp_lex_token.hpp"
#include "wave/cpplexer/re2clex/scanner.hpp"
#include "wave/cpplexer/cpp_lex_interface.hpp"

#include "wave/language_support.hpp"

///////////////////////////////////////////////////////////////////////////////
namespace wave {
namespace cpplexer {
namespace re2clex {

///////////////////////////////////////////////////////////////////////////////
// 
//  encapsulation of the re2c based cpp lexer
//
///////////////////////////////////////////////////////////////////////////////

template <typename IteratorT, typename PositionT = wave::util::file_position_t>
class lexer 
{
public:

    typedef char                                    char_t;
    typedef Scanner                                 base_t;
    typedef wave::cpplexer::lex_token<PositionT>    token_t;
    
    lexer(IteratorT const &first, IteratorT const &last, 
        PositionT const &pos, wave::language_support language);
    ~lexer();

    wave::cpplexer::lex_token<PositionT> get();
    void set_position(PositionT const &pos)
    {
        filename = pos.get_file();
        scanner.line = pos.get_line();
        scanner.file_name = filename.c_str();
    }

// error reporting from the re2c generated lexer
    static int report_error(Scanner *s, char *, ...);

private:
    static char const *tok_names[];
    
    Scanner scanner;
    WAVE_STRINGTYPE filename;
    bool at_eof;
};

///////////////////////////////////////////////////////////////////////////////
// initialize cpp lexer 
template <typename IteratorT, typename PositionT>
inline
lexer<IteratorT, PositionT>::lexer(IteratorT const &first, 
        IteratorT const &last, PositionT const &pos, 
        wave::language_support language) 
:   filename(pos.get_file()), at_eof(false)
{
    memset(&scanner, '\0', sizeof(Scanner));
    scanner.fd = -1;
    scanner.eol_offsets = aq_create();
    scanner.first = scanner.act = (uchar *)&(*first);
    scanner.last = (uchar *)&(*last);
    scanner.line = 1;                   // start with line_no 1
    scanner.error_proc = report_error;
    scanner.file_name = filename.c_str();
    
#if defined(WAVE_SUPPORT_MS_EXTENSIONS)
    scanner.enable_ms_extensions = 1;
#else
    scanner.enable_ms_extensions = 0;
#endif // defined(WAVE_SUPPORT_MS_EXTENSIONS)

#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
    scanner.act_in_c99_mode = wave::need_c99(language);
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    scanner.act_in_cpp0x_mode = wave::need_cpp0x(language);
#endif
#endif

    boost::ignore_unused_variable_warning(language);
}

template <typename IteratorT, typename PositionT>
inline
lexer<IteratorT, PositionT>::~lexer() 
{
    aq_terminate(scanner.eol_offsets);
    free(scanner.bot);
}

///////////////////////////////////////////////////////////////////////////////
//  get the next token from the input stream
template <typename IteratorT, typename PositionT>
inline wave::cpplexer::lex_token<PositionT> 
lexer<IteratorT, PositionT>::get()
{
    using namespace wave::cpplexer::re2clex;
    typedef typename wave::cpplexer::lex_token<PositionT>::string_t string_t;

    if (at_eof) 
        return wave::cpplexer::lex_token<PositionT>();

    token_id id = token_id(scan(&scanner));
    string_t value((char const *)scanner.tok, scanner.cur-scanner.tok);
    
    if (T_IDENTIFIER == id) {
    // test identifier characters for validity (throws if invalid chars found)
        impl::validate_identifier_name(value, scanner.line, -1, filename); 
    }
    else if (T_STRINGLIT == id || T_CHARLIT == id) {
    // test literal characters for validity (throws if invalid chars found)
        impl::validate_literal(value, scanner.line, -1, filename); 
    }
    else if (T_EOF == id) {
    // T_EOF is returned as a valid token, the next call will return T_EOI,
    // i.e. the actual end of input
        at_eof = true;
    }
    return wave::cpplexer::lex_token<PositionT>(id, value, 
        PositionT(filename, scanner.line, -1));
}

template <typename IteratorT, typename PositionT>
inline int 
lexer<IteratorT, PositionT>::report_error(Scanner *s, char *msg, ...)
{
    BOOST_SPIRIT_ASSERT(0 != s);
    BOOST_SPIRIT_ASSERT(0 != msg);

    using namespace std;    // some system have vsprintf in namespace std
    
    char buffer[200];           // should be large enough
    va_list params;
    va_start(params, msg);
    vsprintf(buffer, msg, params);
    va_end(params);
    
    CPPLEXER_THROW(lexing_exception, generic_lexing_error, buffer, s->line, -1, 
        s->file_name);
}

///////////////////////////////////////////////////////////////////////////////
//   
//  lex_functor
//   
///////////////////////////////////////////////////////////////////////////////
     
template <typename IteratorT, typename PositionT = wave::util::file_position_t>
class lex_functor 
:   public lex_input_interface<typename lexer<IteratorT, PositionT>::token_t>
{    
public:

    typedef typename lexer<IteratorT, PositionT>::token_t   token_t;
    
    lex_functor(IteratorT const &first, IteratorT const &last, 
            PositionT const &pos, wave::language_support language)
    :   lexer(first, last, pos, language)
    {}
    virtual ~lex_functor() {}
    
// get the next token from the input stream
    token_t get() { return lexer.get(); }
    void set_position(PositionT const &pos) 
    { lexer.set_position(pos); }

private:
    lexer<IteratorT, PositionT> lexer;
};

///////////////////////////////////////////////////////////////////////////////
//  
//  The new_lexer_gen<>::new_lexer function (declared in cpp_slex_token.hpp)
//  should be defined inline, if the lex_functor shouldn't be instantiated 
//  separately from the lex_iterator.
//
//  Separate (explicit) instantiation helps to reduce compilation time.
//
///////////////////////////////////////////////////////////////////////////////

#if defined(WAVE_SEPARATE_LEXER_INSTANTIATION)
#define WAVE_RE2C_NEW_LEXER_INLINE
#else
#define WAVE_RE2C_NEW_LEXER_INLINE inline
#endif 

}   // namespace re2clex

///////////////////////////////////////////////////////////////////////////////
//
//  The 'new_lexer' function allows the opaque generation of a new lexer object.
//  It is coupled to the iterator type to allow to decouple the lexer/iterator 
//  configurations at compile time.
//
//  This function is declared inside the cpp_slex_token.hpp file, which is 
//  referenced by the source file calling the lexer and the sourcefile, which
//  instantiates the lex_functor. But is is defined here, so it will be 
//  instantiated only while compiling the sourcefile, which instantiates the 
//  lex_functor. While the cpp_re2c_token.hpp file may be included everywhere,
//  this file (cpp_re2c_lexer.hpp) should be included only once. This allows
//  to decouple the lexer interface from the lexer implementation and reduces 
//  compilation time.
//
///////////////////////////////////////////////////////////////////////////////

template <typename IteratorT, typename PositionT>
WAVE_RE2C_NEW_LEXER_INLINE
lex_input_interface<wave::cpplexer::lex_token<PositionT> > *
new_lexer_gen<IteratorT, PositionT>::new_lexer(IteratorT const &first,
    IteratorT const &last, PositionT const &pos, 
    wave::language_support language)
{
    return new re2clex::lex_functor<IteratorT, PositionT>(first, last, pos,
        language);
}

#undef WAVE_RE2C_NEW_LEXER_INLINE

///////////////////////////////////////////////////////////////////////////////
}   // namespace cpplexer
}   // namespace wave
     
#endif // !defined(CPP_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED)

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

Share

About the Author

Hartmut Kaiser

United States United States
Actively involved in Boost and the development of the Spirit parser construction framework.

| Advertise | Privacy | Mobile
Web04 | 2.8.141022.2 | Last Updated 11 Jan 2004
Article Copyright 2003 by Hartmut Kaiser
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid