Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Wave: a Standard conformant C++ preprocessor library

, 10 Jan 2004
Describes a free and fully Standard conformant C++ preprocessor library
wave_preprocessor_demo.zip
wave.exe
wave_preprocessor_demo1.zip
wave.exe
wave_preprocessor_src.zip
wave
doc
theme
bkd.gif
bkd2.gif
bullet.gif
l_arr.gif
l_arr_disabled.gif
r_arr.gif
r_arr_disabled.gif
uc.gif
u_arr.gif
wave.gif
test
boost-build.jam
cpp_tokens
Jamfile.v2
list_includes
project-root.jam
wave
Jamfile
Jamfile.v2
runtests.sh
test_files
wave
cpplexer
re2clex
cpp.re
test
lextest.in
lextest.output
run_tests.sh
test_lexer.in
test_lexer.output
slex
test
lextest.in
lextest.re2c.output
lextest.slex.output
run_tests.sh
grammars
idllexer
re2clex
idl.re
util
wave_preprocessor_src1.zip
bkd.gif
bkd2.gif
bullet.gif
l_arr.gif
l_arr_disabled.gif
r_arr.gif
r_arr_disabled.gif
u_arr.gif
uc.gif
wave.gif
boost-build.jam
Jamfile.v2
project-root.jam
Jamfile
Jamfile.v2
runtests.sh
cpp.re
lextest.in
lextest.output
run_tests.sh
test_lexer.in
test_lexer.output
lextest.in
lextest.re2c.output
lextest.slex.output
run_tests.sh
/*=============================================================================
    Wave: A Standard compliant C++ preprocessor

    Definition of the preprocessor iterator
    
    Copyright (c) 2001-2004 Hartmut Kaiser
    http://spirit.sourceforge.net/

    Use, modification and distribution is subject to the Boost Software
    License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
    http://www.boost.org/LICENSE_1_0.txt)

    See Copyright.txt for full acknowledgements.
=============================================================================*/

#if !defined(CPP_ITERATOR_HPP_175CA88F_7273_43FA_9039_BCF7459E1F29_INCLUDED)
#define CPP_ITERATOR_HPP_175CA88F_7273_43FA_9039_BCF7459E1F29_INCLUDED

#include <string>
#include <vector>
#include <list>
#include <cstdlib>

#include <boost/shared_ptr.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/spirit/core/assert.hpp>
#include <boost/spirit/iterator/multi_pass.hpp>
#include <boost/spirit/tree/parse_tree_utils.hpp>
#include <boost/pool/pool_alloc.hpp>

#include "wave/lex_iterator.hpp"

#include "wave/util/insert_whitespace_detection.hpp"
#include "wave/util/eat_whitespace.hpp"
#include "wave/util/macro_helpers.hpp"
#include "wave/util/interpret_pragma.hpp"
#include "wave/util/transform_iterator.hpp"
#include "wave/util/functor_input.hpp"

#include "wave/grammars/cpp_grammar_gen.hpp"
#include "wave/grammars/cpp_expression_grammar_gen.hpp"
#if defined(WAVE_ENABLE_COMMANDLINE_MACROS)
#include "wave/grammars/cpp_predef_macros_gen.hpp"
#endif // defined(WAVE_ENABLE_COMMANDLINE_MACROS)

#include "wave/cpp_iteration_context.hpp"
#include "wave/cpp_exceptions.hpp"
#include "wave/language_support.hpp"

///////////////////////////////////////////////////////////////////////////////
namespace wave {
namespace util {

///////////////////////////////////////////////////////////////////////////////
// retrieve the macro name from the parse tree
template <typename ParseNodeT, typename TokenT, typename PositionT>
inline void  
retrieve_macroname(ParseNodeT const &node, boost::spirit::parser_id id, 
    TokenT &macroname, PositionT const &act_pos)
{
ParseNodeT const *name_node = 0;

    typedef wave::grammars::cpp_grammar_gen<TokenT> cpp_grammar_t;
    using boost::spirit::find_node;
    if (!find_node(node, id, &name_node)) 
    {
        // ill formed define statement (unexpected, should not happen)
        CPP_THROW(preprocess_exception, bad_define_statement, 
            "bad parse tree (unexpected)", act_pos);
    }
    
typename ParseNodeT::children_t const &children = name_node->children;

    if (0 == children.size() || 
        children[0].value.begin() == children[0].value.end()) 
    {
        // ill formed define statement (unexpected, should not happen)
        CPP_THROW(preprocess_exception, bad_define_statement, 
            "bad parse tree (unexpected)", act_pos);
    }

// retrieve the macro name
    macroname = *children[0].value.begin();
}

///////////////////////////////////////////////////////////////////////////////
// retrieve the macro parameters or the macro definition from the parse tree
template <typename ParseNodeT, typename TokenT, typename ContainerT>
inline bool  
retrieve_macrodefinition(
    ParseNodeT const &node, boost::spirit::parser_id id, 
    ContainerT &macrodefinition, TokenT const &/*t*/)
{
    using namespace WAVE_LEXER_NS;
    typedef typename ParseNodeT::const_tree_iterator const_tree_iterator;

// find macro parameters/macro definition inside the parse tree
std::pair<const_tree_iterator, const_tree_iterator> nodes;

    using boost::spirit::get_node_range;
    if (get_node_range(node, id, nodes)) {
    // copy all parameters to the supplied container
        typename ContainerT::iterator last_nonwhite = macrodefinition.end();
        const_tree_iterator end = nodes.second;
        
        for (const_tree_iterator cit = nodes.first; cit != end; ++cit) {
            if ((*cit).value.begin() != (*cit).value.end()) {
            typename ContainerT::iterator inserted = macrodefinition.insert(
                macrodefinition.end(), *(*cit).value.begin());
                
                if (!IS_CATEGORY(macrodefinition.back(), WhiteSpaceTokenType) &&
                    T_NEWLINE != token_id(macrodefinition.back()) &&
                    T_EOF != token_id(macrodefinition.back()))
                {
                    last_nonwhite = inserted;
                }
            }
        }
        
    // trim trailing whitespace (leading whitespace is trimmed by the grammar)
        if (last_nonwhite != macrodefinition.end()) {
            macrodefinition.erase(++last_nonwhite, macrodefinition.end());
        }
        return true;
    }
    return false;
}

#if defined(WAVE_ENABLE_COMMANDLINE_MACROS)
///////////////////////////////////////////////////////////////////////////////
//  add an additional predefined macro given by a string (MACRO(x)=definition)
template <typename ContextT>
bool add_macro_definition(ContextT &ctx, std::string macrostring,
    bool is_predefined, wave::language_support language)
{
    typedef typename ContextT::token_t token_t;
    typedef typename ContextT::lex_t lex_t;
    typedef typename token_t::position_t position_t;
    typedef wave::grammars::predefined_macros_grammar_gen<token_t> 
        predef_macros_t;

    using namespace WAVE_LEXER_NS;
    
position_t act_pos("command line", 0);
boost::spirit::tree_parse_info<lex_t> hit = 
    predef_macros_t::parse_predefined_macro(
        lex_t(macrostring.begin(), macrostring.end(), position_t(), language), 
        lex_t());

    if (!hit.match || (!hit.full && T_EOF != token_id(*hit.stop))) {
        CPP_THROW(preprocess_exception, bad_macro_definition, macrostring, 
            act_pos);
    }
    
// retrieve the macro definition from the parse tree
token_t macroname;
std::vector<token_t> macroparameters;
typename ContextT::token_sequence_t macrodefinition;
bool has_parameters = false;

    wave::util::retrieve_macroname(*hit.trees.begin(), 
        predef_macros_t::rule_ids.plain_define_id, macroname, act_pos);
    has_parameters = wave::util::retrieve_macrodefinition(*hit.trees.begin(), 
        predef_macros_t::rule_ids.macro_parameters_id, macroparameters, 
        token_t());
    wave::util::retrieve_macrodefinition(*hit.trees.begin(), 
        predef_macros_t::rule_ids.macro_definition_id, macrodefinition,
        token_t());

//  If no macrodefinition is given, and the macro string does not end with a 
//  '=', then the macro should be defined with the value '1'
    if (0 == macrodefinition.size() && 
        '=' != macrostring[macrostring.size()-1])
    {
        macrodefinition.push_back(token_t(T_INTLIT, "1", act_pos));
    }
    
// add the new macro to the macromap
    return ctx.add_macro_definition(macroname, has_parameters, macroparameters, 
        macrodefinition, is_predefined);
}
#endif // defined(WAVE_ENABLE_COMMANDLINE_MACROS)

///////////////////////////////////////////////////////////////////////////////
}   // namespace util

///////////////////////////////////////////////////////////////////////////////
//  forward declaration
template <typename ContextT> class pp_iterator;

namespace impl {

///////////////////////////////////////////////////////////////////////////////
//  
//  pp_iterator_functor
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
class pp_iterator_functor {

public:
// interface to the boost::spirit::multi_pass_policies::functor_input policy
    typedef typename ContextT::token_t              result_type;

//  eof token
    static result_type const eof;

private:
    typedef typename ContextT::lex_t                lex_t;
    typedef typename result_type::string_t          string_t;
    typedef wave::grammars::cpp_grammar_gen<result_type> cpp_grammar_t;

//  iteration context related types (an iteration context represents a current
//  position in an included file)
    typedef base_iteration_context<lex_t>           base_iteration_context_t;
    typedef 
        iteration_context<lex_t, typename ContextT::input_policy_t>
        iteration_context_t;

// parse tree related types
    typedef 
        boost::spirit::node_val_data_factory<boost::spirit::nil_t> 
        node_factory_t;
    typedef 
        boost::spirit::tree_match<lex_t, node_factory_t> 
        parse_tree_match_t;
    typedef typename parse_tree_match_t::node_t         parse_node_t;       // tree_node<node_val_data<> >
    typedef typename parse_tree_match_t::parse_node_t   parse_node_value_t; // node_val_data<>
    typedef typename parse_tree_match_t::container_t    parse_tree_t;       // parse_node_t::children_t

// type of a token sequence
    typedef typename ContextT::token_sequence_t         token_sequence_t;
    
public:
    template <typename IteratorT>
    pp_iterator_functor(ContextT &ctx_, IteratorT const &first_, 
            IteratorT const &last_, typename ContextT::position_t const &pos_,
            wave::language_support language)
    :   ctx(ctx_), 
        iter_ctx(new base_iteration_context_t(
                lex_t(first_, last_, pos_, language), lex_t(), 
                pos_.get_file().c_str()
            )), 
        seen_newline(true), must_emit_line_directive(false),
        act_pos(ctx_.get_main_pos()), last_line(0)
    {
        act_pos.set_file(pos_.get_file());
#if defined(WAVE_SUPPORT_PRAGMA_ONCE)
        ctx_.set_current_filename(pos_.get_file().c_str());
#endif // defined(WAVE_SUPPORT_PRAGMA_ONCE)
    }
    
// get the next preprocessed token
    result_type const &operator()();

// get the last recognized token (for error processing etc.)
    result_type const &current_token() const { return act_token; }

protected:
    friend class pp_iterator<ContextT>;
    void on_include_helper(char const *s, bool is_system);
    
protected:
    result_type const &get_next_token();
    result_type const &pp_token(bool consider_emitting_line_directive = false);

    bool pp_directive();
    bool dispatch_directive(boost::spirit::tree_parse_info<lex_t> const &hit);

    void on_include(string_t const &s, bool is_system);
    void on_include(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);

    void on_define(parse_node_t const &node);
    void on_undefine(result_type const &t);
    
    void on_ifdef(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_ifndef(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_else();
    void on_endif();
    void on_illformed(typename result_type::string_t const &s);
        
    void on_line(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_if(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_elif(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_error(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
#if defined(WAVE_SUPPORT_WARNING_DIRECTIVE)
    void on_warning(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
#endif // defined(WAVE_SUPPORT_WARNING_DIRECTIVE)
    bool on_pragma(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    void on_region(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_endregion();
    void on_import(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    result_type const &emit_line_directive();
    bool returned_from_include();

    bool interpret_pragma(token_sequence_t const &pragma_body,
        token_sequence_t &result);

private:
    ContextT &ctx;              // context, this iterator is associated with
    boost::shared_ptr<base_iteration_context_t> iter_ctx;
    
    bool seen_newline;              // needed for recognizing begin of line
    bool must_emit_line_directive;  // must emit a line directive
    result_type act_token;          // current token
    typename result_type::position_t &act_pos;   // current fileposition (references the macromap)
    int last_line;                  // line number of the previous token
        
    token_sequence_t unput_queue;     // tokens to be preprocessed again
    token_sequence_t pending_queue;   // tokens already preprocessed
    
    // detect whether to insert additional whitespace in between two adjacent 
    // tokens, which otherwise would form a different token type, if 
    // retokenized
    wave::util::insert_whitespace_detection whitespace; 
    
    // remove not needed whitespace from the output stream
    wave::util::eat_whitespace<result_type> eater;
};

///////////////////////////////////////////////////////////////////////////////
//  eof token
template <typename ContextT>
typename pp_iterator_functor<ContextT>::result_type const
    pp_iterator_functor<ContextT>::eof;

///////////////////////////////////////////////////////////////////////////////
//
//  returned_from_include()
// 
//      Tests if it is necessary to pop the include file context (eof inside
//      a file was reached). If yes, it pops this context. Preprocessing will
//      continue with the next outer file scope.
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline bool 
pp_iterator_functor<ContextT>::returned_from_include()
{
    if (iter_ctx->first == iter_ctx->last && ctx.get_iteration_depth() > 0) {
    // call the include policy trace function
        ctx.get_trace_policy().returning_from_include_file();
        
    // restore the previous iteration context after finishing the preprocessing 
    // of the included file
        iter_ctx = ctx.pop_iteration_context();
        
        must_emit_line_directive = true;
        seen_newline = true;

    // restore current file position
        act_pos.set_file(iter_ctx->filename);
#if defined(WAVE_SUPPORT_PRAGMA_ONCE)
        ctx.set_current_filename(iter_ctx->real_filename.c_str());
#endif // defined(WAVE_SUPPORT_PRAGMA_ONCE)

        last_line = iter_ctx->line;
        act_pos.set_line(last_line);
        act_pos.set_column(0);
        
    // restore the actual current directory 
        ctx.set_current_directory(iter_ctx->real_filename.c_str());
        return true;
    }
    return false;
}

///////////////////////////////////////////////////////////////////////////////
//
//  operator()(): get the next preprocessed token
//
//      throws a preprocess_exception, if appropriate
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::operator()()
{
    using namespace WAVE_LEXER_NS;

// loop over skippable whitespace until something significant is found
bool skipped_newline = false;
bool was_seen_newline = seen_newline;

    do {
    // get_next_token assigns result to act_token member
        if (!seen_newline && skipped_newline)
            seen_newline = true;
        get_next_token();
        
    } while (eater.may_skip(act_token, skipped_newline));
    
token_id id = token_id(act_token);
    
// if there were skipped any newline, we must emit a #line directive
    if (was_seen_newline && skipped_newline && 
        !IS_CATEGORY(id, WhiteSpaceTokenType) && 
        !IS_CATEGORY(id, EOLTokenType) && !IS_CATEGORY(id, EOFTokenType)) 
    {
    // must emit a #line directive
        emit_line_directive();
        eater.may_skip(act_token, skipped_newline);     // feed ws eater FSM
        id = token_id(act_token);
    }
    
// cleanup of certain tokens required
    seen_newline = skipped_newline;
    switch (id) {
    case T_NONREPLACABLE_IDENTIFIER:
        act_token.set_token_id(T_IDENTIFIER);
        break;
        
    case T_NEWLINE:
    case T_CPPCOMMENT:
        seen_newline = true;
        ++iter_ctx->emitted_lines;
        break;

    case T_EOF:
        seen_newline = true;
        break;

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    //  The special pp-tokens __comma__, __lparen__ and __rparen__ are to be
    //  converted to their respective 'normal' tokens just before returning
    //  them to the caller.
    case T_COMMA_ALT:
        act_token.set_token_id(T_COMMA);
        act_token.set_value(",");
        break;

    case T_LEFTPAREN_ALT:
        act_token.set_token_id(T_LEFTPAREN);
        act_token.set_value("(");
        break;

    case T_RIGHTPAREN_ALT:
        act_token.set_token_id(T_RIGHTPAREN);
        act_token.set_value(")");
        break;
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    default:
        break;
    }

    if (whitespace.must_insert(id, act_token.get_value())) {
    // must insert some whitespace into the output stream to avoid adjacent
    // tokens, which would form different (and wrong) tokens
        whitespace.shift_tokens(T_SPACE);
        pending_queue.push_front(act_token);        // push this token back
        return act_token = result_type(T_SPACE, 
            typename result_type::string_t(" "), 
            act_token.get_position());
    }
    whitespace.shift_tokens(id);
    return act_token;
}


template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::get_next_token()
{
    using namespace WAVE_LEXER_NS;
    
// if there is something in the unput_queue, then return the next token from
// there (all tokens in the queue are preprocessed already)
    if (pending_queue.size() > 0 || unput_queue.size() > 0) 
        return pp_token();      // return next token
    
// test for EOF, if there is a pending input context, pop it back and continue
// parsing with it
bool returned_from_include_file = returned_from_include();
    
// try to generate the next token 
    if (iter_ctx->first != iter_ctx->last) {
        do {
        // fetch the current token        
            act_token = *iter_ctx->first;

        // adjust the current position (line and column)
        bool was_seen_newline = seen_newline || returned_from_include_file;
        int current_line = act_token.get_position().get_line();
        
            act_pos.set_line(act_pos.get_line() + current_line - last_line);
            act_pos.set_column(act_token.get_position().get_column());
            last_line = current_line;

        // act accordingly on the current token
        token_id id = token_id(act_token);
        
            if (T_EOF == id) {
                if (!seen_newline) {
                // warn, if this file does not end with a newline
                    CPP_THROW(preprocess_exception, last_line_not_terminated, 
                        "", act_pos);
                }
                
            // returned from an include file, continue with the next token
                whitespace.shift_tokens(T_EOF);
                ++iter_ctx->first;
                continue;   // if this is the main file, the while loop breaks
            }
            else if (T_NEWLINE == id || T_CPPCOMMENT == id) {   
            // a newline is to be returned ASAP, a C++ comment too
            // (the C++ comment token includes the trailing newline)
                seen_newline = true;
                ++iter_ctx->first;
                whitespace.shift_tokens(T_NEWLINE);  // whitespace controller
                return act_token; 
            }
            seen_newline = false;

            if (was_seen_newline && pp_directive()) {
            // a pp directive was found
                seen_newline = true;
                must_emit_line_directive = true;

            // loop to the next token to analyze
            // simply fall through, since the iterator was already adjusted 
            // correctly
            }
            else if (ctx.get_if_block_status()) {
            // preprocess this token, eat up more, if appropriate, return 
            // the next preprocessed token
                return pp_token(was_seen_newline);
            }
            else {
            // compilation condition is false: if the current token is a 
            // newline, account for it, otherwise discard the actual token and 
            // try the next one
                if (T_NEWLINE == act_token) {
                    seen_newline = true;
                    must_emit_line_directive = true;
                }

            // next token
                ++iter_ctx->first;
            }
            
        } while (iter_ctx->first != iter_ctx->last || returned_from_include());
    }
    
    if (returned_from_include_file) {
    // if there was an '#include' statement on the last line of the main file 
    // we have to return an additional newline token
        seen_newline = true;
        
        whitespace.shift_tokens(T_NEWLINE);  // whitespace controller
        return act_token = result_type(T_NEWLINE, 
            typename result_type::string_t("\n"), 
            cpp_grammar_t::pos_of_newline);
    }
    
// overall eof reached
    if (ctx.get_if_block_depth() > 0) {
    // missing endif directive(s)
        CPP_THROW(preprocess_exception, missing_matching_endif, "", act_pos);
    }

    whitespace.shift_tokens(T_EOF);     // whitespace controller
    return act_token = eof;             // return eof token
}

///////////////////////////////////////////////////////////////////////////////
//
//  emit_line_directive(): emits a line directive from the act_token data
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::emit_line_directive()
{
    using namespace WAVE_LEXER_NS;
    
typename ContextT::position_t pos = act_token.get_position();

    if (iter_ctx->emitted_lines != act_pos.get_line()) {
    // unput the current token
        pending_queue.push_front(act_token);
        pos.set_line(act_pos.get_line());

        if (iter_ctx->emitted_lines+1 == act_pos.get_line()) {
        // prefer to output a single newline instead of the #line directive
            whitespace.shift_tokens(T_NEWLINE);
            act_token = result_type(T_NEWLINE, "\n", pos);
        }
        else {
        // account for the here emitted newline
            act_pos.set_line(act_pos.get_line()-1);
            iter_ctx->emitted_lines = act_pos.get_line();
            --last_line;
        
        // the #line directive has to be pushed back into the pending queue in 
        // reverse order

        // unput the complete #line directive
        std::string file("\"");
        boost::filesystem::path filename(act_pos.get_file().c_str(), 
            boost::filesystem::native);
        
            using wave::util::impl::escape_lit;
            file += escape_lit(filename.native_file_string()) + "\"";
            pending_queue.push_front(result_type(T_NEWLINE, "\n", pos));
            pending_queue.push_front(result_type(T_STRINGLIT, file.c_str(), pos));
            pending_queue.push_front(result_type(T_SPACE, " ", pos));
            
        // 21 is the max required size for a 64 bit integer represented as a 
        // string
        char buffer[22];

            using namespace std;    // for some systems sprintf is in namespace std
            sprintf (buffer, "%d", pos.get_line());
            pending_queue.push_front(result_type(T_INTLIT, buffer, pos));
            pending_queue.push_front(result_type(T_SPACE, " ", pos));
            
        // return the #line token itself
            whitespace.shift_tokens(T_PP_LINE);
            act_token = result_type(T_PP_LINE, "#line", pos);
        }
    }

// we are now in sync
    must_emit_line_directive = false;
    return act_token;
}

///////////////////////////////////////////////////////////////////////////////
//
//  pptoken(): return the next preprocessed token
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::pp_token(bool consider_emitting_line_directive)
{
    using namespace WAVE_LEXER_NS;

token_id id = token_id(*iter_ctx->first);

    // eat all T_PLACEHOLDER tokens, eventually slipped through out of the
    // macro engine
    do { 
        if (!pending_queue.empty()) {
        // if there are pending tokens in the queue, return the first one
            act_token = pending_queue.front();
            pending_queue.pop_front();
        }
        else if (!unput_queue.empty() 
            || T_IDENTIFIER == id || IS_CATEGORY(id, KeywordTokenType) 
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
            || (T_COLON_COLON == id && wave::need_cpp0x(ctx.get_language()))
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
            ) 
        {
        //  call the lexer, preprocess the required number of tokens, put them
        //  into the unput queue
            act_token = ctx.expand_tokensequence(iter_ctx->first, 
                iter_ctx->last, pending_queue, unput_queue);
        }
        else {
        // simply return the next token
            act_token = *iter_ctx->first;
            ++iter_ctx->first;
        }
        id = token_id(act_token);
        
    } while (T_PLACEHOLDER == id);

    return act_token;
}

///////////////////////////////////////////////////////////////////////////////
//
//  pp_directive(): recognize a preprocessor directive
//
///////////////////////////////////////////////////////////////////////////////
namespace {

    template <typename IteratorT>
    bool next_token_is_pp_directive(IteratorT &it, IteratorT const &end)
    {
        using namespace WAVE_LEXER_NS;
        
        token_id id = T_ANY;
        for (/**/; it != end; ++it) {
            id = token_id(*it);
            if (!IS_CATEGORY(id, WhiteSpaceTokenType))
                break;          // skip leading whitespace
            if (IS_CATEGORY(id, EOLTokenType))
                break;          // do not enter a new line
        }
        BOOST_SPIRIT_ASSERT(it == end || id != T_ANY);
        return it != end && IS_CATEGORY(id, PPTokenType);
    }
    
    template <typename IteratorT>
    bool is_pp_null(IteratorT &it, IteratorT const &end)
    {
        using namespace WAVE_LEXER_NS;
        
        BOOST_SPIRIT_ASSERT(T_POUND == token_id(*it));
        for (++it; it != end; ++it) {
        token_id id = token_id(*it);
        
            if (T_CPPCOMMENT == id || T_NEWLINE == id) {
                ++it;           // skip eol/C++ comment
                return true;    // found pp_null
            }

            if (!IS_CATEGORY(id, WhiteSpaceTokenType))
                break;
        }
        return false;
    }

    template <typename IteratorT>
    bool skip_to_eol(IteratorT &it, IteratorT const &end)
    {
        using namespace WAVE_LEXER_NS;
        
        for (/**/; it != end; ++it) {
        token_id id = token_id(*it);
        
            if (T_CPPCOMMENT == id || T_NEWLINE == id) {
                ++it;           // skip eol/C++ comment
                return true;    // found pp_null
            }
        }
        return false;
    }
}

template <typename ContextT> 
inline bool
pp_iterator_functor<ContextT>::pp_directive()
{
    using namespace cpplexer;
    
// test, if the next non-whitespace token is a pp directive
lex_t it = iter_ctx->first;

    if (!next_token_is_pp_directive(it, iter_ctx->last)) {
    // eventually skip null pp directive (no need to do it via the parser)
        if (it != iter_ctx->last && T_POUND == token_id(*it) &&
            is_pp_null(it, iter_ctx->last))
        {
            seen_newline = true;
            iter_ctx->first = it;   // start over with the next line
            return true;
        }
    }
    
    if (it == iter_ctx->last)
        return false;

// ignore all pp directives not related to conditional compilation while
// if block status is false
    if (!ctx.get_if_block_status() && 
        !IS_EXTCATEGORY(*it, PPConditionalTokenType))
    {
        seen_newline = true;
        skip_to_eol(it, iter_ctx->last);
        iter_ctx->first = it;       // start over with the next line
        return true;
    }

// found a pp directive, so try to identify it, start with the pp_token
bool found_eof = false;
boost::spirit::tree_parse_info<lex_t> hit = 
    cpp_grammar_t::parse_cpp_grammar(it, iter_ctx->last, found_eof, act_pos);

    if (hit.match) {
    // position the iterator past the matched sequence to allow 
    // resynchronisation, if an error occurs
        iter_ctx->first = hit.stop;
        
    // found a valid pp directive, dispatch to the correct function to handle 
    // the found pp directive
    bool result = dispatch_directive (hit);
    
        if (found_eof) {
        // The line was terminated with an end of file token.
        // So trigger a warning, that the last line was not terminated with a 
        // newline.
            CPP_THROW(preprocess_exception, last_line_not_terminated, "", 
                act_pos);
        }
        return result;
    }
    return false;
}

///////////////////////////////////////////////////////////////////////////////
//
//  dispatch_directive(): dispatch a recognized preprocessor directive
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline bool
pp_iterator_functor<ContextT>::dispatch_directive(
    boost::spirit::tree_parse_info<lex_t> const &hit)
{
    using namespace cpplexer;
    using namespace boost::spirit;
    
    typedef typename parse_tree_t::const_iterator const_child_iterator_t;
    
// this iterator points to the root node of the parse tree
const_child_iterator_t begin = hit.trees.begin();

// decide, which preprocessor directive was found
parse_tree_t const &root = (*begin).children;
parse_node_value_t const &nodeval = get_first_leaf(*root.begin()).value;
//long node_id = nodeval.id().to_long();

const_child_iterator_t begin_child_it = (*root.begin()).children.begin();
const_child_iterator_t end_child_it = (*root.begin()).children.end();

//token_id id = cpp_grammar_t::found_directive;

    switch (cpp_grammar_t::found_directive) {
    case T_PP_QHEADER:      // #include "..."
        on_include ((*nodeval.begin()).get_value(), false);
        break;

    case T_PP_HHEADER:      // #include <...>
        on_include ((*nodeval.begin()).get_value(), true);
        break;
    
    case T_PP_INCLUDE:      // #include ...
        on_include (begin_child_it, end_child_it);
        break;

    case T_PP_DEFINE:       // #define
        on_define (*begin);
        break;

    case T_PP_UNDEF:        // #undef
        on_undefine(*nodeval.begin());
        break;

    case T_PP_IFDEF:        // #ifdef
        on_ifdef(begin_child_it, end_child_it);
        break;

    case T_PP_IFNDEF:       // #ifndef
        on_ifndef(begin_child_it, end_child_it);
        break;

    case T_PP_IF:           // #if
        on_if(begin_child_it, end_child_it);
        break;

    case T_PP_ELIF:         // #elif
        on_elif(begin_child_it, end_child_it);
        break;

    case T_PP_ELSE:         // #else
        on_else();
        break;

    case T_PP_ENDIF:        // #endif
        on_endif();
        break;

    case T_PP_LINE:         // #line
        on_line(begin_child_it, end_child_it);
        break;
        
    case T_PP_ERROR:        // #error
        on_error(begin_child_it, end_child_it);
        break;

#if defined(WAVE_SUPPORT_WARNING_DIRECTIVE)
    case T_PP_WARNING:      // #warning
        on_warning(begin_child_it, end_child_it);
        break;
#endif // defined(WAVE_SUPPORT_WARNING_DIRECTIVE)

    case T_PP_PRAGMA:       // #pragma
        return on_pragma(begin_child_it, end_child_it);

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    case T_PP_REGION:       // #region
        if (wave::need_cpp0x(ctx.get_language())) {
        // C++0x mode is enabled
        const_child_iterator_t begin_it = (*begin_child_it).children.begin();
        const_child_iterator_t end_it = (*begin_child_it).children.end();
        
            on_region(begin_it, end_it);
        }
        else {
            on_illformed("#" WAVE_PP_REGION);
        }
        break;
        
    case T_PP_ENDREGION:    // #endregion
        if (wave::need_cpp0x(ctx.get_language())) {
        // C++0x mode is enabled
            on_endregion();
        }
        else {
            on_illformed("#" WAVE_PP_ENDREGION);
        }
        break;

    case T_PP_IMPORT:       // #import
        if (wave::need_cpp0x(ctx.get_language())) {
        // C++0x mode is enabled
        const_child_iterator_t begin_it = (*begin_child_it).children.begin();
        const_child_iterator_t end_it = (*begin_child_it).children.end();
        
            on_import(begin_it, end_it);
        }
        else {
            on_illformed("#" WAVE_PP_IMPORT);
        }
        break;
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    default:                // #something else
        on_illformed((*nodeval.begin()).get_value());
        break;
    }
    return true;    // return newline only
}

///////////////////////////////////////////////////////////////////////////////
// 
//  on_include: handle #include <...> or #include "..." directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_include (string_t const &s, bool is_system) 
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// strip quotes first, extract filename
typename string_t::size_type pos_end = s.find_last_of(is_system ? '>' : '\"');

    if (string_t::npos == pos_end) {
        CPP_THROW(preprocess_exception, bad_include_statement, s, act_pos);
    }

typename string_t::size_type pos_begin = 
    s.find_last_of(is_system ? '<' : '\"', pos_end-1);

    if (string_t::npos == pos_begin) {
        CPP_THROW(preprocess_exception, bad_include_statement, s, act_pos);
    }

std::string file_path(s.substr(pos_begin+1, pos_end-pos_begin-1).c_str());

// finally include the file
    on_include_helper(file_path.c_str(), is_system);
}
       
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_include_helper (
    char const *s, bool is_system) 
{
    namespace fs = boost::filesystem;

// try to locate the given file, searching through the include path lists
std::string file_path(s);

    if (!ctx.find_include_file (file_path, is_system)) {
        CPP_THROW(preprocess_exception, bad_include_file, file_path, act_pos);
    }

fs::path native_path(file_path, fs::native);

    if (!fs::exists(native_path)) {
        CPP_THROW(preprocess_exception, bad_include_file, file_path, act_pos);
    }

// test, if this file is known through a #pragma once directive
#if defined(WAVE_SUPPORT_PRAGMA_ONCE)
    if (!ctx.has_pragma_once(native_path.native_file_string())) 
#endif // defined(WAVE_SUPPORT_PRAGMA_ONCE)
    {
    // the new include file determines the actual current directory
        ctx.set_current_directory(file_path.c_str());
        
    // preprocess the opened file
    boost::shared_ptr<base_iteration_context_t> new_iter_ctx (
        new iteration_context_t(native_path.native_file_string().c_str(), 
            act_pos, ctx.get_language()));

    // call the include policy trace function
        ctx.get_trace_policy().opened_include_file(file_path,
            ctx.get_iteration_depth(), is_system);

    // store current file position
        iter_ctx->filename = act_pos.get_file();
        iter_ctx->line = act_pos.get_line();
        
    // push the old iteration context onto the stack and continue with the new
        ctx.push_iteration_context(act_pos, iter_ctx);
        iter_ctx = new_iter_ctx;
        seen_newline = true;        // fake a newline to trigger pp_directive
        must_emit_line_directive = true;
        
        act_pos.set_file(iter_ctx->filename);  // initialize file position
#if defined(WAVE_SUPPORT_PRAGMA_ONCE)
        ctx.set_current_filename(iter_ctx->real_filename.c_str());
#endif // defined(WAVE_SUPPORT_PRAGMA_ONCE)

        last_line = iter_ctx->line;
        act_pos.set_line(last_line);
        act_pos.set_column(0);
    }
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_include(): handle #include ... directives
//
///////////////////////////////////////////////////////////////////////////////

template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_include(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// preprocess the given token sequence (the body of the #include directive)
get_token_value<result_type, parse_node_t> get_value;
token_sequence_t expanded;
token_sequence_t toexpand;

    std::copy(make_ref_transform_iterator(begin, get_value), 
        make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename token_sequence_t::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded, 
        false);

// now, include the file
string_t s (trim_whitespace(wave::util::impl::as_string(expanded)));
bool is_system = '<' == s[0] && '>' == s[s.size()-1];

    if (!is_system && !('\"' == s[0] && '\"' == s[s.size()-1])) {
    // should resolve into something like <...> or "..."
        CPP_THROW(preprocess_exception, bad_include_statement, 
            s, act_pos);
    }
    on_include(s, is_system);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_define(): handle #define directives
//
///////////////////////////////////////////////////////////////////////////////

template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_define (parse_node_t const &node) 
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// retrieve the macro definition from the parse tree
result_type macroname;
std::vector<result_type> macroparameters;
token_sequence_t macrodefinition;
bool has_parameters = false;

    wave::util::retrieve_macroname(node, 
        cpp_grammar_t::rule_ids.plain_define_id, macroname, 
        act_token.get_position());
    has_parameters = wave::util::retrieve_macrodefinition(node, 
        cpp_grammar_t::rule_ids.macro_parameters_id, macroparameters, act_token);
    wave::util::retrieve_macrodefinition(node, 
        cpp_grammar_t::rule_ids.macro_definition_id, macrodefinition, act_token);

    if (has_parameters) {
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
        if (wave::need_variadics(ctx.get_language())) {
        // test whether ellipsis are given, and if yes, if these are placed as the
        // last argument
            using namespace cpplexer;
            typedef typename std::vector<result_type>::iterator 
                parameter_iterator_t;
            
            bool seen_ellipses = false;
            parameter_iterator_t end = macroparameters.end();
            for (parameter_iterator_t pit = macroparameters.begin(); 
                pit != end; ++pit) 
            {
                if (seen_ellipses) {
                // ellipses are not the last given formal argument
                    CPP_THROW(preprocess_exception, bad_define_statement, 
                        macroname.get_value(), (*pit).get_position());
                }
                if (T_ELLIPSIS == token_id(*pit)) 
                    seen_ellipses = true;
            }
            
        // if there wasn't an ellipsis, then there shouldn't be a __VA_ARGS__ 
        // placeholder in the definition too [C99 Standard 6.10.3.5]
            if (!seen_ellipses) {
                typedef typename token_sequence_t::iterator definition_iterator_t;

                bool seen_va_args = false;
                definition_iterator_t end = macrodefinition.end();
                for (definition_iterator_t dit = macrodefinition.begin(); 
                    dit != end; ++dit) 
                {
                    if (T_IDENTIFIER == token_id(*dit) && 
                        "__VA_ARGS__" == (*dit).get_value())
                    {
                        seen_va_args = true;
                    }
                }
                if (seen_va_args) {
                // must not have seen __VA_ARGS__ placeholder
                    CPP_THROW(preprocess_exception, bad_define_statement, 
                        macroname.get_value(), act_token.get_position());
                }
            }
        }
        else
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
#if !defined(WAVE_USE_RE2C_IDL_LEXER)
        {
        // test, that there is no T_ELLIPSES given
            using namespace cpplexer;
            typedef typename std::vector<result_type>::iterator 
                parameter_iterator_t;
            
            parameter_iterator_t end = macroparameters.end();
            for (parameter_iterator_t pit = macroparameters.begin(); 
                pit != end; ++pit) 
            {
                if (T_ELLIPSIS == token_id(*pit)) {
                // if variadics are disabled, no ellipses should be given
                    CPP_THROW(preprocess_exception, bad_define_statement, 
                        macroname.get_value(), (*pit).get_position());
                }
            }
        }
#endif // !defined(WAVE_USE_RE2C_IDL_LEXER)
    }
    
// add the new macro to the macromap
    ctx.add_macro_definition(macroname, has_parameters, macroparameters, 
        macrodefinition);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_define(): handle #undef directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_undefine (result_type const &token) 
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// retrieve the macro name to undefine from the parse tree
    ctx.remove_macro_definition(token.get_value()); // throws for predefined macros
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_ifdef(): handle #ifdef directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_ifdef(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
get_token_value<result_type, parse_node_t> get_value;
bool is_defined = ctx.is_defined_macro(
        make_ref_transform_iterator((*begin).children.begin(), get_value), 
        make_ref_transform_iterator((*begin).children.end(), get_value));

    ctx.enter_if_block(is_defined);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_ifndef(): handle #ifndef directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_ifndef(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
get_token_value<result_type, parse_node_t> get_value;
bool is_defined = ctx.is_defined_macro(
        make_ref_transform_iterator((*begin).children.begin(), get_value), 
        make_ref_transform_iterator((*begin).children.end(), get_value));

    ctx.enter_if_block(!is_defined);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_else(): handle #else directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_else()
{
    if (!ctx.enter_else_block()) {
    // #else without matching #if
        CPP_THROW(preprocess_exception, missing_matching_if, "#else", 
            act_pos);
    }
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_endif(): handle #endif directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_endif()
{
    if (!ctx.exit_if_block()) {
    // #endif without matching #if
        CPP_THROW(preprocess_exception, missing_matching_if, "#endif", 
            act_pos);
    }
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_if(): handle #if directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_if(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
// preprocess the given sequence into the provided list
get_token_value<result_type, parse_node_t> get_value;
token_sequence_t expanded;
token_sequence_t toexpand;

    std::copy(make_ref_transform_iterator(begin, get_value), 
        make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename token_sequence_t::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded);

    typename token_sequence_t::iterator exp_end = expanded.end();
    for (typename token_sequence_t::iterator exp_it = expanded.begin();
         exp_it != exp_end; ++exp_it)
    {
        using namespace WAVE_LEXER_NS;
        
        token_id id = token_id(*exp_it);
        if (IS_CATEGORY(id, IdentifierTokenType) ||
            IS_CATEGORY(id, KeywordTokenType))
        {
            (*exp_it).set_token_id(T_INTLIT);
            (*exp_it).set_value("0");
        }
    }
    
#if defined(WAVE_DUMP_CONDITIONAL_EXPRESSIONS)
    {
        string_t outstr(wave::util::impl::as_string(toexpand));
        outstr += "(" + wave::util::impl::as_string(expanded) + ")";
        WAVE_DUMP_CONDITIONAL_EXPRESSIONS_OUT << "#if " << outstr << std::endl;
    }
#endif // defined(WAVE_DUMP_CONDITIONAL_EXPRESSIONS)

// parse the expression and enter the #if block
    ctx.enter_if_block(grammars::expression_grammar_gen<result_type>::
            evaluate(expanded.begin(), expanded.end(), act_pos,
                ctx.get_if_block_status()));
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_elif(): handle #elif directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_elif(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    if (ctx.get_if_block_status()) {
        if (!ctx.enter_elif_block(false)) {
        // #else without matching #if
            CPP_THROW(preprocess_exception, missing_matching_if, "#elif", 
                act_pos);
        }
        return;     // #if/previous #elif was true, so don't enter this #elif 
    }
            
// preprocess the given sequence into the provided list
get_token_value<result_type, parse_node_t> get_value;
token_sequence_t expanded;
token_sequence_t toexpand;

    std::copy(make_ref_transform_iterator(begin, get_value), 
        make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename token_sequence_t::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded);
    
    typename token_sequence_t::iterator exp_end = expanded.end();
    for (typename token_sequence_t::iterator exp_it = expanded.begin();
         exp_it != exp_end; ++exp_it)
    {
        using namespace WAVE_LEXER_NS;
        
        token_id id = token_id(*exp_it);
        if (IS_CATEGORY(id, IdentifierTokenType) ||
            IS_CATEGORY(id, KeywordTokenType))
        {
            (*exp_it).set_token_id(T_INTLIT);
            (*exp_it).set_value("0");
        }
    }

#if defined(WAVE_DUMP_CONDITIONAL_EXPRESSIONS)
    {
        string_t outstr(wave::util::impl::as_string(toexpand));
        outstr += "(" + wave::util::impl::as_string(expanded) + ")";
        WAVE_DUMP_CONDITIONAL_EXPRESSIONS_OUT << "#elif " << outstr << std::endl;
    }
#endif // defined(WAVE_DUMP_CONDITIONAL_EXPRESSIONS)

// parse the expression and enter the #elif block
    ctx.enter_elif_block(grammars::expression_grammar_gen<result_type>::
            evaluate(expanded.begin(), expanded.end(), act_pos,
                ctx.get_if_block_status()));
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_illformed(): handles the illegal directive
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_illformed(
    typename result_type::string_t const &s)
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());
    CPP_THROW(preprocess_exception, ill_formed_directive, s, act_pos);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_line(): handle #line directives
//
///////////////////////////////////////////////////////////////////////////////

namespace {

    template <typename IteratorT, typename StringT>
    bool retrieve_line_info (IteratorT first, IteratorT const &last,
        int &line, StringT &file)
    {
        using namespace WAVE_LEXER_NS;
        if (T_INTLIT == token_id(*first)) {
        // extract line number
            using namespace std;    // some system have atoi in namespace std
            line = atoi((*first).get_value().c_str());
            
        // extract file name (if it is given)
            while (++first != last && IS_CATEGORY(*first, WhiteSpaceTokenType)) 
                /**/;   // skip whitespace
                
            if (first != last) {
                if (T_STRINGLIT != token_id(*first)) 
                    return false;

                StringT const &file_lit = (*first).get_value();
                file = file_lit.substr(1, file_lit.size()-2);
            }
            return true;
        }
        return false;
    }
}

template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_line(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// Try to extract the line number and file name from the given token list
// directly. If that fails, preprocess the whole token sequence and try again 
// to extract this information.
token_sequence_t expanded;
get_token_value<result_type, parse_node_t> get_value;

    typedef typename ref_transform_iterator_generator<
            get_token_value<result_type, parse_node_t>, 
            typename parse_tree_t::const_iterator
        >::type const_tree_iterator_t;
        
const_tree_iterator_t first = make_ref_transform_iterator(begin, get_value);
const_tree_iterator_t last = make_ref_transform_iterator(end, get_value);
    
// try to interpret the #line body as a number followed by an optional
// string literal
int line = 0;
string_t file_name;

    if (!retrieve_line_info(first, last, line, file_name)) {
    // preprocess the body of this #line message
    token_sequence_t toexpand;

        std::copy(first, make_ref_transform_iterator(end, get_value),
            std::inserter(toexpand, toexpand.end()));

        typename token_sequence_t::iterator begin2 = toexpand.begin();
        ctx.expand_whole_tokensequence(begin2, toexpand.end(), 
            expanded, false);
            
        if (!retrieve_line_info(expanded.begin(), expanded.end(), line, 
            file_name))
        {
            CPP_THROW(preprocess_exception, bad_line_statement, 
                wave::util::impl::as_string(expanded), act_pos)
        }
    }
    
// the queues should be empty at this point
    BOOST_SPIRIT_ASSERT(unput_queue.empty());
    BOOST_SPIRIT_ASSERT(pending_queue.empty());

    if (!file_name.empty())     // reuse current file name 
        act_pos.set_file(file_name.c_str());
    act_pos.set_line(line-1);
    last_line = act_token.get_position().get_line();
    
    must_emit_line_directive = true;
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_error(): handle #error directives
//
///////////////////////////////////////////////////////////////////////////////
namespace {

    // trim all whitespace from the begin and the end of the given string
    template <typename StringT>
    inline StringT 
    trim_whitespace(StringT const &s)
    {
        typedef typename StringT::size_type size_type;
        
        size_type first = s.find_first_not_of(" \t\v\f");
        if (StringT::npos == first)
            return StringT();
        size_type last = s.find_last_not_of(" \t\v\f");
        return s.substr(first, last-first+1);
    }
}

template <typename ContextT> 
inline void 
pp_iterator_functor<ContextT>::on_error(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// preprocess the given sequence into the provided list
token_sequence_t expanded;
get_token_value<result_type, parse_node_t> get_value;

typename ref_transform_iterator_generator<
        get_token_value<result_type, parse_node_t>, 
        typename parse_tree_t::const_iterator
    >::type first = make_ref_transform_iterator(begin, get_value);
    
#if defined(WAVE_PREPROCESS_ERROR_MESSAGE_BODY)
// preprocess the body of this #warning message
token_sequence_t toexpand;

    std::copy(first, make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename token_sequence_t::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded, 
        false);
#else
// simply copy the body of this #warning message to the issued diagnostic
// message
    std::copy(first, make_ref_transform_iterator(end, get_value), 
        std::inserter(expanded, expanded.end()));
#endif // defined(WAVE_PREPROCESS_ERROR_MESSAGE_BODY)

// throw the corresponding exception
    CPP_THROW(preprocess_exception, error_directive, 
        wave::util::impl::as_string(expanded), act_pos);
}

#if defined(WAVE_SUPPORT_WARNING_DIRECTIVE)
///////////////////////////////////////////////////////////////////////////////
//  
//  on_warning(): handle #warning directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void 
pp_iterator_functor<ContextT>::on_warning(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// preprocess the given sequence into the provided list
token_sequence_t expanded;
get_token_value<result_type, parse_node_t> get_value;

typename ref_transform_iterator_generator<
        get_token_value<result_type, parse_node_t>, 
        typename parse_tree_t::const_iterator
    >::type first = make_ref_transform_iterator(begin, get_value);
    
#if defined(WAVE_PREPROCESS_ERROR_MESSAGE_BODY)
// preprocess the body of this #warning message
token_sequence_t toexpand;

    std::copy(first, make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename token_sequence_t::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded, 
        false);
#else
// simply copy the body of this #warning message to the issued diagnostic
// message
    std::copy(first, make_ref_transform_iterator(end, get_value), 
        std::inserter(expanded, expanded.end()));
#endif // defined(WAVE_PREPROCESS_ERROR_MESSAGE_BODY)

// throw the corresponding exception
    CPP_THROW(preprocess_exception, warning_directive, 
        wave::util::impl::as_string(expanded), act_pos);
}
#endif // defined(WAVE_SUPPORT_WARNING_DIRECTIVE)

///////////////////////////////////////////////////////////////////////////////
//  
//  on_pragma(): handle #pragma directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline bool
pp_iterator_functor<ContextT>::on_pragma(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    using namespace WAVE_LEXER_NS;
    
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// Look at the pragma token sequence and decide, if the first token is STDC
// (see C99 standard [6.10.6.2]), if it is, the sequence must _not_ be
// preprocessed.
token_sequence_t expanded;
get_token_value<result_type, parse_node_t> get_value;

    typedef typename ref_transform_iterator_generator<
            get_token_value<result_type, parse_node_t>, 
            typename parse_tree_t::const_iterator
        >::type const_tree_iterator_t;
        
const_tree_iterator_t first = make_ref_transform_iterator(begin, get_value);
const_tree_iterator_t last = make_ref_transform_iterator(end, get_value);

    expanded.push_back(result_type(T_PP_PRAGMA, "#pragma", act_token.get_position()));
    expanded.push_back(result_type(T_SPACE, " ", act_token.get_position()));
        
    while (++first != last && IS_CATEGORY(*first, WhiteSpaceTokenType)) 
        expanded.push_back(*first);   // skip whitespace

    if (first != last) {
        if (T_IDENTIFIER == token_id(*first) && 
            wave::need_c99(ctx.get_language()) && 
            (*first).get_value() == "STDC") 
        {
        // do _not_ preprocess the token sequence
            std::copy(first, last, std::inserter(expanded, expanded.end()));
        }
        else {
#if defined(WAVE_PREPROCESS_PRAGMA_BODY)
        // preprocess the given tokensequence
        token_sequence_t toexpand;

            std::copy(first, last, std::inserter(toexpand, toexpand.end()));

            typename token_sequence_t::iterator begin2 = toexpand.begin();
            ctx.expand_whole_tokensequence(begin2, toexpand.end(), 
                expanded, false);
#else
        // do _not_ preprocess the token sequence
            std::copy(first, last, std::inserter(expanded, expanded.end()));
#endif // defined(WAVE_PREPROCESS_PRAGMA_BODY)
        }
    }
    expanded.push_back(result_type(T_NEWLINE, "\n", act_token.get_position()));
        
// the queues should be empty at this point
    BOOST_SPIRIT_ASSERT(unput_queue.empty());
    BOOST_SPIRIT_ASSERT(pending_queue.empty());

// try to interpret the expanded #pragma body 
    token_sequence_t pending;
    if (interpret_pragma(expanded, pending)) {
    // if there is some replacement text, insert it into the pending queue
        if (pending.size() > 0)
            pending_queue.splice(pending_queue.begin(), pending);
        return true;        // this #pragma was successfully recognized
    }
    
#if defined(WAVE_EMIT_PRAGMA_DIRECTIVES)
// Move the resulting token sequence into the pending_queue, so it will be 
// returned to the caller.
    pending_queue.splice(pending_queue.begin(), expanded);
    return false;           // return the whole #pragma directive
#else
    return true;            // skip the #pragma at all
#endif // defined(WAVE_EMIT_PRAGMA_DIRECTIVES)
}

template <typename ContextT> 
inline bool
pp_iterator_functor<ContextT>::interpret_pragma(
    token_sequence_t const &pragma_body, token_sequence_t &result)
{
    using namespace cpplexer;
    
    typename token_sequence_t::const_iterator end = pragma_body.end();
    typename token_sequence_t::const_iterator it = pragma_body.begin();
    for (++it; it != end && IS_CATEGORY(*it, WhiteSpaceTokenType); ++it) 
        /**/;   // skip whitespace
    
    if (it == end)      // eof reached
        return false;

    return wave::util::interpret_pragma(ctx, act_token, it, end, result,
        ctx.get_language());
}

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
///////////////////////////////////////////////////////////////////////////////
//  
//  on_region(): handle #region directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void
pp_iterator_functor<ContextT>::on_region(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

//  if there is a name given, start a named region
    if (begin != end) {
    // (re-)open named region
    get_token_value<result_type, parse_node_t> get_value;
    
        ctx.begin_scope(
            make_ref_transform_iterator((*begin).children.begin(), get_value),
            make_ref_transform_iterator((*begin).children.end(), get_value));
    }
    else {
    // (re-)open unnamed region
        ctx.begin_unnamed_scope();
    }
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_endregion(): handle #endregion directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void
pp_iterator_functor<ContextT>::on_endregion()
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());
    ctx.end_scope();
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_import(): handle #import directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void
pp_iterator_functor<ContextT>::on_import(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    BOOST_SPIRIT_ASSERT(ctx.get_if_block_status());

// import all given macro names/scope names
    get_token_value<result_type, parse_node_t> get_value;
    for (typename parse_tree_t::const_iterator it = begin; it != end; ++it) {
    // import every single of the listed names    
        ctx.import_name(
            make_ref_transform_iterator((*it).children.begin(), get_value), 
            make_ref_transform_iterator((*it).children.end(), get_value));
    }
}

#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

///////////////////////////////////////////////////////////////////////////////
}   // namespace impl

///////////////////////////////////////////////////////////////////////////////
//  
//  pp_iterator
//
//      The wave::pp_iterator template is the iterator, through which
//      the resulting preprocessed input stream is accessible.
//  
///////////////////////////////////////////////////////////////////////////////

template <typename ContextT>
class pp_iterator 
:   public boost::spirit::multi_pass<
        wave::impl::pp_iterator_functor<ContextT>,
        wave::util::functor_input
    >
{
public:
    typedef wave::impl::pp_iterator_functor<ContextT> input_policy_t;

private:
    typedef 
        boost::spirit::multi_pass<input_policy_t, wave::util::functor_input>
        base_t;
    typedef pp_iterator<ContextT> self_t;
    typedef wave::util::functor_input functor_input_t;
    
public:
    pp_iterator() 
    {}
    
    template <typename IteratorT>
    pp_iterator(ContextT &ctx, IteratorT const &first, IteratorT const &last,
        typename ContextT::position_t const &pos, 
        wave::language_support language)
    :   base_t(input_policy_t(ctx, first, last, pos, language))
    {}
    
    void force_include(char const *path_, bool is_last)
    { 
        get_functor().on_include_helper(path_, false); 
        if (is_last) {
            this->functor_input_t::
                template inner<input_policy_t>::advance_input();
        }
    }
};

///////////////////////////////////////////////////////////////////////////////
}   // namespace wave

#endif // !defined(CPP_ITERATOR_HPP_175CA88F_7273_43FA_9039_BCF7459E1F29_INCLUDED)

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

Share

About the Author

Hartmut Kaiser

United States United States
Actively involved in Boost and the development of the Spirit parser construction framework.

| Advertise | Privacy | Terms of Use | Mobile
Web02 | 2.8.141220.1 | Last Updated 11 Jan 2004
Article Copyright 2003 by Hartmut Kaiser
Everything else Copyright © CodeProject, 1999-2014
Layout: fixed | fluid