Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Wave: a Standard conformant C++ preprocessor library

, 10 Jan 2004
Describes a free and fully Standard conformant C++ preprocessor library
wave_preprocessor_demo.zip
wave.exe
wave_preprocessor_demo1.zip
wave.exe
wave_preprocessor_src.zip
wave
doc
theme
bkd.gif
bkd2.gif
bullet.gif
l_arr.gif
l_arr_disabled.gif
r_arr.gif
r_arr_disabled.gif
uc.gif
u_arr.gif
wave.gif
test
boost-build.jam
cpp_tokens
Jamfile.v2
list_includes
project-root.jam
wave
Jamfile
Jamfile.v2
runtests.sh
test_files
wave
cpplexer
re2clex
cpp.re
test
lextest.in
lextest.output
run_tests.sh
test_lexer.in
test_lexer.output
slex
test
lextest.in
lextest.re2c.output
lextest.slex.output
run_tests.sh
grammars
idllexer
re2clex
idl.re
util
wave_preprocessor_src1.zip
bkd.gif
bkd2.gif
bullet.gif
l_arr.gif
l_arr_disabled.gif
r_arr.gif
r_arr_disabled.gif
u_arr.gif
uc.gif
wave.gif
boost-build.jam
Jamfile.v2
project-root.jam
Jamfile
Jamfile.v2
runtests.sh
cpp.re
lextest.in
lextest.output
run_tests.sh
test_lexer.in
test_lexer.output
lextest.in
lextest.re2c.output
lextest.slex.output
run_tests.sh
/*=============================================================================
    Wave: A Standard compliant C++ preprocessor

    Definition of the preprocessor iterator
    
    Copyright (c) 2001-2003 Hartmut Kaiser
    http://spirit.sourceforge.net/

    Permission to copy, use, modify, sell and distribute this software
    is granted provided this copyright notice appears in all copies.
    This software is provided "as is" without express or implied
    warranty, and with no claim as to its suitability for any purpose.

    See Copyright.txt for full copyright notices and acknowledgements.
=============================================================================*/

#if !defined(CPP_ITERATOR_HPP_175CA88F_7273_43FA_9039_BCF7459E1F29_INCLUDED)
#define CPP_ITERATOR_HPP_175CA88F_7273_43FA_9039_BCF7459E1F29_INCLUDED

#include <string>
#include <vector>
#include <list>
#include <cstdlib>

#include <boost/shared_ptr.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/spirit/core/assert.hpp>
#include <boost/spirit/iterator/multi_pass.hpp>
#include <boost/spirit/tree/parse_tree_utils.hpp>

#include "wave/cpplexer/cpp_lex_iterator.hpp"

#include "wave/util/insert_whitespace_detection.hpp"
//#include "wave/util/eat_whitespace.hpp"
#include "wave/util/macro_helpers.hpp"
#include "wave/util/interpret_pragma.hpp"
#include "wave/util/transform_iterator.hpp"

#include "wave/grammars/cpp_grammar_gen.hpp"
#include "wave/grammars/cpp_expression_grammar_gen.hpp"
#if defined(WAVE_ENABLE_COMMANDLINE_MACROS)
#include "wave/grammars/cpp_predef_macros_gen.hpp"
#endif // defined(WAVE_ENABLE_COMMANDLINE_MACROS)

#include "wave/cpp_iteration_context.hpp"
#include "wave/cpp_exceptions.hpp"
#include "wave/language_support.hpp"

///////////////////////////////////////////////////////////////////////////////
namespace wave {
namespace util {

///////////////////////////////////////////////////////////////////////////////
// retrieve the macro name from the parse tree
template <typename ParseNodeT, typename TokenT, typename PositionT>
inline void  
retrieve_macroname(ParseNodeT const &node, boost::spirit::parser_id id, 
    TokenT &macroname, PositionT const &act_pos)
{
ParseNodeT const *name_node = 0;

    typedef wave::grammars::cpp_grammar_gen<TokenT> cpp_grammar_t;
    using boost::spirit::find_node;
    if (!find_node(node, id, &name_node)) 
    {
        // ill formed define statement (unexpected, should not happen)
        CPP_THROW(preprocess_exception, bad_define_statement, 
            "bad parse tree (unexpected)", act_pos);
    }
    
typename ParseNodeT::children_t const &children = name_node->children;

    if (0 == children.size() || 
        children[0].value.begin() == children[0].value.end()) 
    {
        // ill formed define statement (unexpected, should not happen)
        CPP_THROW(preprocess_exception, bad_define_statement, 
            "bad parse tree (unexpected)", act_pos);
    }

// retrieve the macro name
    macroname = *children[0].value.begin();
}

///////////////////////////////////////////////////////////////////////////////
// retrieve the macro parameters or the macro definition from the parse tree
template <typename ParseNodeT, typename TokenT, typename ContainerT>
inline bool  
retrieve_macrodefinition(
    ParseNodeT const &node, boost::spirit::parser_id id, 
    ContainerT &macrodefinition, TokenT const &/*t*/)
{
    using namespace wave::cpplexer;
    typedef typename ParseNodeT::const_tree_iterator const_tree_iterator;

// find macro parameters/macro definition inside the parse tree
std::pair<const_tree_iterator, const_tree_iterator> nodes;

    using boost::spirit::get_node_range;
    if (get_node_range(node, id, nodes)) {
    // copy all parameters to the supplied container
        typename ContainerT::iterator last_nonwhite = macrodefinition.end();
        const_tree_iterator end = nodes.second;
        
        for (const_tree_iterator cit = nodes.first; cit != end; ++cit) {
            if ((*cit).value.begin() != (*cit).value.end()) {
            typename ContainerT::iterator inserted = macrodefinition.insert(
                macrodefinition.end(), *(*cit).value.begin());
                
                if (!IS_CATEGORY(macrodefinition.back(), WhiteSpaceTokenType) &&
                    T_NEWLINE != token_id(macrodefinition.back()))
                {
                    last_nonwhite = inserted;
                }
            }
        }
        
    // trim trailing whitespace (leading whitespace is trimmed by the grammar)
        if (last_nonwhite != macrodefinition.end()) {
            macrodefinition.erase(++last_nonwhite, macrodefinition.end());
        }
        return true;
    }
    return false;
}

#if defined(WAVE_ENABLE_COMMANDLINE_MACROS)
///////////////////////////////////////////////////////////////////////////////
//  add an additional predefined macro given by a string (MACRO(x)=definition)
template <typename ContextT>
bool add_macro_definition(ContextT &ctx, std::string macrostring,
    bool is_predefined, wave::language_support language)
{
    typedef typename ContextT::token_t token_t;
    typedef typename ContextT::lex_t lex_t;
    typedef typename token_t::position_t position_t;
    typedef wave::grammars::predefined_macros_grammar_gen<token_t> 
        predef_macros_t;

    using namespace wave::cpplexer;
    
position_t act_pos("command line", 0);
boost::spirit::tree_parse_info<lex_t> hit = 
    predef_macros_t::parse_predefined_macro(
        lex_t(macrostring.begin(), macrostring.end(), position_t(), language), 
        lex_t());

    if (!hit.match || (!hit.full && T_EOF != token_id(*hit.stop))) {
        CPP_THROW(preprocess_exception, bad_macro_definition, macrostring, 
            act_pos);
    }
    
// retrieve the macro definition from the parse tree
token_t macroname;
std::vector<token_t> macroparameters;
std::list<token_t> macrodefinition;
bool has_parameters = false;

    wave::util::retrieve_macroname(*hit.trees.begin(), 
        predef_macros_t::rule_ids.plain_define_id, macroname, act_pos);
    has_parameters = wave::util::retrieve_macrodefinition(*hit.trees.begin(), 
        predef_macros_t::rule_ids.macro_parameters_id, macroparameters, 
        token_t());
    wave::util::retrieve_macrodefinition(*hit.trees.begin(), 
        predef_macros_t::rule_ids.macro_definition_id, macrodefinition,
        token_t());

//  If no macrodefinition is given, and the macro string does not end with a 
//  '=', then the macro should be defined with the value '1'
    if (0 == macrodefinition.size() && 
        '=' != macrostring[macrostring.size()-1])
    {
        macrodefinition.push_back(token_t(T_INTLIT, "1", act_pos));
    }
    
// add the new macro to the macromap
    return ctx.add_macro_definition(macroname, has_parameters, macroparameters, 
        macrodefinition, is_predefined);
}
#endif // defined(WAVE_ENABLE_COMMANDLINE_MACROS)

///////////////////////////////////////////////////////////////////////////////
}   // namespace util

///////////////////////////////////////////////////////////////////////////////
//  forward declaration
template <typename ContextT> class pp_iterator;

namespace impl {

///////////////////////////////////////////////////////////////////////////////
//  
//  pp_iterator_functor
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
class pp_iterator_functor {

public:
// interface to the boost::spirit::multi_pass_policies::functor_input policy
    typedef typename ContextT::token_t              result_type;

//  eof token
    static result_type const eof;

private:
    typedef typename ContextT::lex_t                lex_t;
    typedef typename result_type::string_t          string_t;
    typedef wave::grammars::cpp_grammar_gen<result_type> cpp_grammar_t;

//  iteration context related types (an iteration context represents a current
//  position in an included file)
    typedef base_iteration_context<lex_t>           base_iteration_context_t;
    typedef 
        iteration_context<lex_t, typename ContextT::input_policy_t>
        iteration_context_t;

// parse tree related types
    typedef 
        boost::spirit::node_val_data_factory<boost::spirit::nil_t> 
        node_factory_t;
    typedef 
        boost::spirit::tree_match<lex_t, node_factory_t> 
        parse_tree_match_t;
    typedef typename parse_tree_match_t::node_t         parse_node_t;       // tree_node<node_val_data<> >
    typedef typename parse_tree_match_t::parse_node_t   parse_node_value_t; // node_val_data<>
    typedef typename parse_tree_match_t::container_t    parse_tree_t;       // parse_node_t::children_t

public:
    template <typename IteratorT>
    pp_iterator_functor(ContextT &ctx_, IteratorT const &first_, 
            IteratorT const &last_, typename ContextT::position_t const &pos_,
            wave::language_support language)
    :   ctx(ctx_), 
        iter_ctx(new base_iteration_context_t(
                lex_t(first_, last_, pos_, language), lex_t(), 
                pos_.get_file().c_str()
            )), 
        seen_newline(true), must_emit_line_directive(false),
        act_pos(ctx_.get_main_pos()), last_line(0)
    {
        act_pos.set_file(pos_.get_file());
    }
    
// get the next preprocessed token
    result_type const &operator()();

// get the last recognized token (for error processing etc.)
    result_type const &current_token() const { return act_token; }
    
protected:
    friend class pp_iterator<ContextT>;
    void on_include_helper(char const *s, bool is_system);
    
protected:
    result_type const &pp_token(bool consider_emitting_line_directive = false);
    bool pp_directive();
    bool dispatch_directive(boost::spirit::tree_parse_info<lex_t> const &hit);

    void on_include(string_t const &s, bool is_system);
    void on_include(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);

    void on_define(parse_node_t const &node);
    void on_undefine(result_type const &t);
    
    void on_ifdef(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_ifndef(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_else();
    void on_endif();
    void on_illformed(typename result_type::string_t const &s);
        
    void on_line(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_if(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_elif(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_error(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
#if defined(WAVE_SUPPORT_WARNING_DIRECTIVE)
    void on_warning(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
#endif // defined(WAVE_SUPPORT_WARNING_DIRECTIVE)
    bool on_pragma(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    void on_region(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
    void on_endregion();
    void on_import(typename parse_tree_t::const_iterator const &begin,
        typename parse_tree_t::const_iterator const &end);
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    result_type const &emit_line_directive();
    bool returned_from_include();

    bool interpret_pragma(std::list<result_type> const &pragma_body,
        std::list<result_type> &result);

private:
    ContextT &ctx;              // context, this iterator is associated with
    boost::shared_ptr<base_iteration_context_t> iter_ctx;
    
    bool seen_newline;              // needed for recognizing begin of line
    bool must_emit_line_directive;  // must emit a line directive
    result_type act_token;          // current token
    typename result_type::position_t &act_pos;   // current fileposition (references the macromap)
    int last_line;                  // line number of the previous token
        
    std::list<result_type> unput_queue;     // tokens to be preprocessed again
    std::list<result_type> pending_queue;   // tokens already preprocessed
    
    // detect whether to insert additional whitespace in between two adjacent 
    // tokens, which otherwise would form a different token type, if 
    // retokenized
    wave::util::insert_whitespace_detection whitespace; 
    
    // remove not needed whitespace from the output stream
//    wave::util::eat_whitespace<result_type> eater;
};

///////////////////////////////////////////////////////////////////////////////
//  eof token
template <typename ContextT>
typename pp_iterator_functor<ContextT>::result_type const
    pp_iterator_functor<ContextT>::eof;

///////////////////////////////////////////////////////////////////////////////
//
//  returned_from_include()
// 
//      Tests if it is nessecary to pop the include file context (eof inside
//      a file was reached). If yes, it pops this context. Preprocessing will
//      continue with the next outer file scope.
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline bool 
pp_iterator_functor<ContextT>::returned_from_include()
{
    if (iter_ctx->first == iter_ctx->last && ctx.get_iteration_depth() > 0) {
    // restore the previous iteration context after finishing the preprocessing 
    // of the included file
        iter_ctx = ctx.pop_iteration_context();
        
        must_emit_line_directive = true;
        seen_newline = true;

    // restore current file position
        act_pos.set_file(iter_ctx->filename);
        last_line = iter_ctx->line;
        act_pos.set_line(last_line);
        act_pos.set_column(0);
        
    // restore the actual current directory 
        ctx.set_current_directory(iter_ctx->filename.c_str());
        return true;
    }
    return false;
}

///////////////////////////////////////////////////////////////////////////////
//
//  operator()(): get the next preprocessed token
//
//      throws a preprocess_exception, if appropriate
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::operator()()
{
    using namespace wave::cpplexer;

// if there is something in the unput_queue, then return the next token from
// there (all tokens in the queue are preprocessed already)
    if (pending_queue.size() > 0 || unput_queue.size() > 0) 
        return pp_token();      // return next token
    
// test for EOI, if there is a pending input context, pop it back and continue
// parsing with it
bool returned_from_include_file = returned_from_include();
    
// try to generate the next token 
    if (iter_ctx->first != iter_ctx->last) {
        do {
        bool was_seen_newline = seen_newline || returned_from_include_file;
    
            act_token = *iter_ctx->first;
            
            act_pos.set_line(act_pos.get_line() + 
                act_token.get_position().get_line() - last_line);
            act_pos.set_column(act_token.get_position().get_column());
            last_line = act_token.get_position().get_line();
            
            if (T_EOF == act_token && !seen_newline) {
            // warn, if this file does not end with a newline
                CPP_THROW(preprocess_exception, last_line_not_terminated, "", 
                    act_pos);
            }
            else if (T_NEWLINE == act_token || T_CPPCOMMENT == act_token) {   
            // a newline is to be returned ASAP, a C++ comment too
            // (the C++ comment token includes the trailing newline)
                seen_newline = true;
                
//            bool skipped_newline = false;
//            
//                ++iter_ctx->first;
//                if (!eater.may_skip(act_token, skipped_newline)) {
//                    ++iter_ctx->emitted_lines;
//                    whitespace.shift_tokens(T_NEWLINE);  // whitespace controller
//                    return act_token;
//                }
//                if (skipped_newline)
//                    must_emit_line_directive = true;
//                continue;   // next token

                ++iter_ctx->emitted_lines;
                ++iter_ctx->first;
                whitespace.shift_tokens(T_NEWLINE);  // whitespace controller
                return act_token; 
            }
            seen_newline = false;

            if (was_seen_newline && pp_directive()) {
            // a pp directive was found
                seen_newline = true;
                must_emit_line_directive = true;

            // loop to the next token to analyze
            // simply fall through, since the iterator was already adjusted 
            // correctly
            }
            else if (ctx.get_if_block_status()) {
            // preprocess this token, eat up more, if appropriate, return 
            // the next preprocessed token
                return pp_token(was_seen_newline);
            }
            else {
            // compilation condition is false: if the current token is a 
            // newline, account for it, otherwise discard the actual token and 
            // try the next one
                if (T_NEWLINE == act_token) {
                    seen_newline = true;
                    must_emit_line_directive = true;
                }

            // next token
                ++iter_ctx->first;
            }
            
        } while (iter_ctx->first != iter_ctx->last || returned_from_include());
    }
    
    if (returned_from_include_file) {
    // if there was an '#include' statement on the last line of the main file 
    // we have to return an additional newline token
        seen_newline = true;
        ++iter_ctx->emitted_lines;
        
        whitespace.shift_tokens(T_NEWLINE);  // whitespace controller
        return act_token = result_type(T_NEWLINE, 
            typename result_type::string_t("\n"), 
            cpp_grammar_t::pos_of_newline);
    }
    
// overall eof reached
    if (ctx.get_if_block_depth() > 0) {
    // missing endif directive(s)
        CPP_THROW(preprocess_exception, missing_matching_endif, "", act_pos);
    }

    whitespace.shift_tokens(T_EOF);     // whitespace controller
    return act_token = eof;             // return eof token
}

///////////////////////////////////////////////////////////////////////////////
//
//  emit_line_directive(): emits a line directive from the act_token data
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::emit_line_directive()
{
    using namespace wave::cpplexer;
    
typename ContextT::position_t pos = act_token.get_position();

    if (iter_ctx->emitted_lines != act_pos.get_line()) {
    // unput the current token
        pending_queue.push_front(act_token);
        pos.set_line(act_pos.get_line());

        if (iter_ctx->emitted_lines+1 == act_pos.get_line()) {
        // prefer to output a single newline instead of the #line directive
            iter_ctx->emitted_lines = act_pos.get_line();
            whitespace.shift_tokens(T_NEWLINE);
            act_token = result_type(T_NEWLINE, "\n", pos);
        }
        else {
        // account for the here emitted newline
            act_pos.set_line(act_pos.get_line()-1);
            iter_ctx->emitted_lines = act_pos.get_line();
            --last_line;
            
        // unput the complete #line diective
        std::string file("\"");
        boost::filesystem::path filename(act_pos.get_file().c_str(), 
            boost::filesystem::native);
        
            using wave::util::impl::escape_lit;
            file += escape_lit(filename.native_file_string()) + "\"";
            pending_queue.push_front(result_type(T_NEWLINE, "\n", pos));
            pending_queue.push_front(result_type(T_STRINGLIT, file.c_str(), pos));
            pending_queue.push_front(result_type(T_SPACE, " ", pos));
            
        // 21 is the max required size for a 64 bit integer represented as a 
        // string
        char buffer[22];

            using namespace std;    // for some systems sprintf is in namespace std
            sprintf (buffer, "%d", pos.get_line());
            pending_queue.push_front(result_type(T_INTLIT, buffer, pos));
            pending_queue.push_front(result_type(T_SPACE, " ", pos));
            
        // return the #line token itself
            whitespace.shift_tokens(T_PP_LINE);
            act_token = result_type(T_PP_LINE, "#line", pos);
        }
    }

// we are now in sync
    must_emit_line_directive = false;
    return act_token;
}

///////////////////////////////////////////////////////////////////////////////
//
//  pptoken(): return the next preprocessed token
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline typename pp_iterator_functor<ContextT>::result_type const &
pp_iterator_functor<ContextT>::pp_token(bool consider_emitting_line_directive)
{
    using namespace wave::cpplexer;

token_id id = token_id(*iter_ctx->first);
//bool skipped_newline = false;

    // eat all T_PLACEHOLDER tokens, eventually slipped through out of the
    // macro engine
    do { 
        if (!pending_queue.empty()) {
        // if there are pending tokens in the queue, return the first one
            act_token = pending_queue.front();
            pending_queue.pop_front();
        }
        else if (!unput_queue.empty() || 
            T_IDENTIFIER == id || IS_CATEGORY(id, KeywordTokenType)) 
        {
        //  call the lexer, preprocess the required number of tokens, put them
        //  into the unput queue
            act_token = ctx.expand_tokensequence(iter_ctx->first, 
                iter_ctx->last, pending_queue, unput_queue);
        }
        else {
        // simply return the next token
            act_token = *iter_ctx->first;
            ++iter_ctx->first;
        }
        id = token_id(act_token);
        
    } while (T_PLACEHOLDER == id /*|| eater.may_skip(act_token, skipped_newline)*/);

// if there were skipped any newline, we must emit a #line directive
//bool preset_seen_newline = false;
//
//    if (skipped_newline) {
//        preset_seen_newline = true;
//        must_emit_line_directive = true;
//    }
    
    if (T_EOF != id && (consider_emitting_line_directive || seen_newline) &&
        must_emit_line_directive) 
    {
    // must emit a #line directive
        emit_line_directive();
        id = token_id(act_token);
    }
    
    if (whitespace.must_insert(id, act_token.get_value())) {
    // must insert some whitespace into the output stream to avoid adjacent
    // tokens, which would form different (and wrong) tokens
        whitespace.shift_tokens(T_SPACE);
        pending_queue.push_front(act_token);        // push this token back
        return act_token = result_type(T_SPACE, 
            typename result_type::string_t(" "), 
            act_token.get_position());
    }

// cleanup of certain tokens required
//    seen_newline = preset_seen_newline;
    seen_newline = false;
    switch (id) {
    case T_NONREPLACABLE_IDENTIFIER:
        act_token.set_token_id(T_IDENTIFIER);
        break;
        
    case T_NEWLINE:
    case T_CPPCOMMENT:
    case T_EOF:
        seen_newline = true;
        ++iter_ctx->emitted_lines;
        break;

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    //  The special pp-tokens __comma__, __lparen__ and __rparen__ are to be
    //  converted to their respective 'normal' tokens just before returning
    //  them to the caller.
    case T_COMMA_ALT:
        act_token.set_token_id(T_COMMA);
        act_token.set_value(",");
        break;

    case T_LEFTPAREN_ALT:
        act_token.set_token_id(T_LEFTPAREN);
        act_token.set_value("(");
        break;

    case T_RIGHTPAREN_ALT:
        act_token.set_token_id(T_RIGHTPAREN);
        act_token.set_value(")");
        break;
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    default:
        break;
    }

    whitespace.shift_tokens(id);
    return act_token;
}

///////////////////////////////////////////////////////////////////////////////
//
//  pp_directive(): recognize a preprocessor directive
//
///////////////////////////////////////////////////////////////////////////////
namespace {

    template <typename IteratorT>
    bool next_token_is_pp_directive(IteratorT &it, IteratorT const &end)
    {
        using namespace cpplexer;
        for (/**/; it != end; ++it) {
            if (!IS_CATEGORY(*it, WhiteSpaceTokenType))
                break;          // skip leading whitespace
            if (IS_CATEGORY(*it, EOLTokenType))
                break;          // do not enter a new line
        }
        return it != end && IS_CATEGORY(*it, PPTokenType);
    }
    
    template <typename IteratorT>
    bool is_pp_null(IteratorT &it, IteratorT const &end)
    {
        using namespace cpplexer;
        
        BOOST_SPIRIT_ASSERT(T_POUND == token_id(*it));
        for (++it; it != end; ++it) {
            if (T_CPPCOMMENT == token_id(*it) || T_NEWLINE == token_id(*it)) {
                ++it;           // skip eol/C++ comment
                return true;    // found pp_null
            }

            if (!IS_CATEGORY(*it, WhiteSpaceTokenType))
                break;
        }
        return false;
    }

    template <typename IteratorT>
    bool skip_to_eol(IteratorT &it, IteratorT const &end)
    {
        using namespace cpplexer;
        for (/**/; it != end; ++it) {
            if (T_CPPCOMMENT == token_id(*it) || T_NEWLINE == token_id(*it)) {
                ++it;           // skip eol/C++ comment
                return true;    // found pp_null
            }
        }
        return false;
    }
}

template <typename ContextT> 
inline bool
pp_iterator_functor<ContextT>::pp_directive()
{
    using namespace cpplexer;
    
// test, if the next non-whitespace token is a pp directive
lex_t it = iter_ctx->first;

    if (!next_token_is_pp_directive(it, iter_ctx->last)) {
    // eventually skip null pp directive (no need to do it via the parser)
        if (it != iter_ctx->last && T_POUND == token_id(*it) &&
            is_pp_null(it, iter_ctx->last))
        {
            iter_ctx->first = it;   // start over with the next line
            return true;
        }
    }
    
    if (it == iter_ctx->last)
        return false;

// ignore all pp directives not related to conditional compilation while
// if block status is false
    if (!ctx.get_if_block_status() && 
        !IS_EXTCATEGORY(*it, PPConditionalTokenType))
    {
        skip_to_eol(it, iter_ctx->last);
        iter_ctx->first = it;       // start over with the next line
        return true;
    }

// found a pp directive, so try to identify it, start with the pp_token
bool found_eof = false;
boost::spirit::tree_parse_info<lex_t> hit = 
    cpp_grammar_t::parse_cpp_grammar(it, iter_ctx->last, found_eof, act_pos);

    if (hit.match) {
    // position the iterator past the matched sequence to allow 
    // resynchronisation, if an error occurs
        iter_ctx->first = hit.stop;
        
    // found a valid pp directive, dispatch to the correct function to handle 
    // the found pp directive
    bool result = dispatch_directive (hit);
    
        if (found_eof) {
        // The line was terminated with an end of file token.
        // So trigger a warning, that the last line was not terminated with a 
        // newline.
            CPP_THROW(preprocess_exception, last_line_not_terminated, "", 
                act_pos);
        }
        return result;
    }
    return false;
}

///////////////////////////////////////////////////////////////////////////////
//
//  dispatch_directive(): dispatch a recognized preprocessor directive
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline bool
pp_iterator_functor<ContextT>::dispatch_directive(
    boost::spirit::tree_parse_info<lex_t> const &hit)
{
    using namespace cpplexer;
    using namespace boost::spirit;
    
    typedef typename parse_tree_t::const_iterator const_child_iterator_t;
    
// this iterator points to the root node of the parse tree
const_child_iterator_t begin = hit.trees.begin();

// decide, which preprocessor directive was found
parse_tree_t const &root = (*begin).children;
parse_node_value_t const &nodeval = get_first_leaf(*root.begin()).value;
//long node_id = nodeval.id().to_long();

const_child_iterator_t begin_child_it = (*root.begin()).children.begin();
const_child_iterator_t end_child_it = (*root.begin()).children.end();

//token_id id = cpp_grammar_t::found_directive;

    switch (cpp_grammar_t::found_directive) {
    case T_PP_QHEADER:      // #include "..."
        on_include ((*nodeval.begin()).get_value(), false);
        break;

    case T_PP_HHEADER:      // #include <...>
        on_include ((*nodeval.begin()).get_value(), true);
        break;
    
    case T_PP_INCLUDE:      // #include ...
        on_include (begin_child_it, end_child_it);
        break;

    case T_PP_DEFINE:       // #define
        on_define (*begin);
        break;

    case T_PP_UNDEF:        // #undef
        on_undefine(*nodeval.begin());
        break;

    case T_PP_IFDEF:        // #ifdef
        on_ifdef(begin_child_it, end_child_it);
        break;

    case T_PP_IFNDEF:       // #ifndef
        on_ifndef(begin_child_it, end_child_it);
        break;

    case T_PP_IF:           // #if
        on_if(begin_child_it, end_child_it);
        break;

    case T_PP_ELIF:         // #elif
        on_elif(begin_child_it, end_child_it);
        break;

    case T_PP_ELSE:         // #else
        on_else();
        break;

    case T_PP_ENDIF:        // #endif
        on_endif();
        break;

    case T_PP_LINE:         // #line
        on_line(begin_child_it, end_child_it);
        break;
        
    case T_PP_ERROR:        // #error
        on_error(begin_child_it, end_child_it);
        break;

#if defined(WAVE_SUPPORT_WARNING_DIRECTIVE)
    case T_PP_WARNING:      // #warning
        on_warning(begin_child_it, end_child_it);
        break;
#endif // defined(WAVE_SUPPORT_WARNING_DIRECTIVE)

    case T_PP_PRAGMA:       // #pragma
        return on_pragma(begin_child_it, end_child_it);

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    case T_PP_REGION:       // #region
        if (wave::need_cpp0x(ctx.get_language())) {
        // C++0x mode is enabled
        const_child_iterator_t begin_it = (*begin_child_it).children.begin();
        const_child_iterator_t end_it = (*begin_child_it).children.end();
        
            on_region(begin_it, end_it);
        }
        else {
            on_illformed("#" WAVE_PP_REGION);
        }
        break;
        
    case T_PP_ENDREGION:    // #endregion
        if (wave::need_cpp0x(ctx.get_language())) {
        // C++0x mode is enabled
            on_endregion();
        }
        else {
            on_illformed("#" WAVE_PP_ENDREGION);
        }
        break;

    case T_PP_IMPORT:       // #import
        if (wave::need_cpp0x(ctx.get_language())) {
        // C++0x mode is enabled
        const_child_iterator_t begin_it = (*begin_child_it).children.begin();
        const_child_iterator_t end_it = (*begin_child_it).children.end();
        
            on_import(begin_it, end_it);
        }
        else {
            on_illformed("#" WAVE_PP_IMPORT);
        }
        break;
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    default:                // #something else
        on_illformed((*nodeval.begin()).get_value());
        break;
    }
    return true;    // return newline only
}

///////////////////////////////////////////////////////////////////////////////
// 
//  on_include: handle #include <...> or #include "..." directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_include (string_t const &s, bool is_system) 
{
//  skip this include, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

// strip quotes first, extract filename
typename string_t::size_type pos_end = s.find_last_of(is_system ? '>' : '\"');

    if (string_t::npos == pos_end) {
        CPP_THROW(preprocess_exception, bad_include_statement, s, act_pos);
    }

typename string_t::size_type pos_begin = 
    s.find_last_of(is_system ? '<' : '\"', pos_end-1);

    if (string_t::npos == pos_begin) {
        CPP_THROW(preprocess_exception, bad_include_statement, s, act_pos);
    }

std::string file_path(s.substr(pos_begin+1, pos_end-pos_begin-1).c_str());

// finally include the file
    on_include_helper(file_path.c_str(), is_system);
}
       
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_include_helper (
    char const *s, bool is_system) 
{
    namespace fs = boost::filesystem;

// try to locate the given file, searching through the include path lists
std::string file_path(s);

    if (!ctx.find_include_file (file_path, is_system)) {
        CPP_THROW(preprocess_exception, bad_include_file, file_path, act_pos);
    }

fs::path native_path(file_path, fs::native);

    if (!fs::exists(native_path)) {
        CPP_THROW(preprocess_exception, bad_include_file, file_path, act_pos);
    }

// the new include file determines the actual current directory
    ctx.set_current_directory(file_path.c_str());
    
// preprocess the opened file
boost::shared_ptr<base_iteration_context_t> new_iter_ctx (
    new iteration_context_t(native_path.native_file_string().c_str(), act_pos,
        ctx.get_language()));

// store current file position
    iter_ctx->filename = act_pos.get_file();
    iter_ctx->line = act_pos.get_line();
    
// push the old iteration context onto the stack and continue with the new
    ctx.push_iteration_context(act_pos, iter_ctx);
    iter_ctx = new_iter_ctx;
    seen_newline = true;            // fake a newline to trigger pp_directive
    must_emit_line_directive = true;
    
    act_pos.set_file(iter_ctx->filename);  // initialize file position
    last_line = iter_ctx->line;
    act_pos.set_line(last_line);
    act_pos.set_column(0);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_include(): handle #include ... directives
//
///////////////////////////////////////////////////////////////////////////////

template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_include(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
//  skip this include, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

// preprocess the given token sequence (the body of the #include directive)
get_token_value<result_type, parse_node_t> get_value;
std::list<result_type> expanded;
std::list<result_type> toexpand;

    std::copy(make_ref_transform_iterator(begin, get_value), 
        make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename std::list<result_type>::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded, 
        false);

// now, include the file
string_t s (trim_whitespace(wave::util::impl::as_string(expanded)));
bool is_system = '<' == s[0] && '>' == s[s.size()-1];

    if (!is_system && !('\"' == s[0] && '\"' == s[s.size()-1])) {
    // should resolve into something like <...> or "..."
        CPP_THROW(preprocess_exception, bad_include_statement, 
            s, act_pos);
    }
    on_include(s, is_system);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_define(): handle #define directives
//
///////////////////////////////////////////////////////////////////////////////

template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_define (parse_node_t const &node) 
{
//  skip this define, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

// retrieve the macro definition from the parse tree
result_type macroname;
std::vector<result_type> macroparameters;
std::list<result_type> macrodefinition;
bool has_parameters = false;

    wave::util::retrieve_macroname(node, 
        cpp_grammar_t::rule_ids.plain_define_id, macroname, 
        act_token.get_position());
    has_parameters = wave::util::retrieve_macrodefinition(node, 
        cpp_grammar_t::rule_ids.macro_parameters_id, macroparameters, act_token);
    wave::util::retrieve_macrodefinition(node, 
        cpp_grammar_t::rule_ids.macro_definition_id, macrodefinition, act_token);

#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
    if (has_parameters && wave::need_variadics(ctx.get_language())) {
    // test whether ellipsis are given, and if yes, if these are placed as the
    // last argument
        using namespace cpplexer;
        typedef typename std::vector<result_type>::iterator 
            parameter_iterator_t;
        
        bool seen_ellipses = false;
        parameter_iterator_t end = macroparameters.end();
        for (parameter_iterator_t pit = macroparameters.begin(); 
             pit != end; ++pit) 
        {
            if (seen_ellipses) {
            // ellipses are not the last given formal argument
                CPP_THROW(preprocess_exception, bad_define_statement, 
                    macroname.get_value(), act_token.get_position());
            }
            if (T_ELLIPSIS == token_id(*pit)) 
                seen_ellipses = true;
        }
        
    // if there wasn't an ellipsis, then there shouldn't be a __VA_ARGS__ 
    // placeholder in the definition too [C99 Standard 6.10.3.5]
        if (!seen_ellipses) {
            typedef typename std::list<result_type>::iterator 
                definition_iterator_t;

            bool seen_va_args = false;
            definition_iterator_t end = macrodefinition.end();
            for (definition_iterator_t dit = macrodefinition.begin(); 
                 dit != end; ++dit) 
            {
                if (T_IDENTIFIER == token_id(*dit) && 
                    "__VA_ARGS__" == (*dit).get_value())
                {
                    seen_va_args = true;
                }
            }
            if (seen_va_args) {
            // must not have seen __VA_ARGS__ placeholder
                CPP_THROW(preprocess_exception, bad_define_statement, 
                    macroname.get_value(), act_token.get_position());
            }
        }
    }
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)

// add the new macro to the macromap
    ctx.add_macro_definition(macroname, has_parameters, macroparameters, 
        macrodefinition);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_define(): handle #undef directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_undefine (result_type const &token) 
{
//  skip this undefine, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

// retrieve the macro name to undefine from the parse tree
    ctx.remove_macro_definition(token.get_value()); // throws for predefined macros
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_ifdef(): handle #ifdef directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_ifdef(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
get_token_value<result_type, parse_node_t> get_value;
bool is_defined = ctx.is_defined_macro(
        make_ref_transform_iterator((*begin).children.begin(), get_value), 
        make_ref_transform_iterator((*begin).children.end(), get_value));

    ctx.enter_if_block(is_defined);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_ifndef(): handle #ifndef directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_ifndef(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
get_token_value<result_type, parse_node_t> get_value;
bool is_defined = ctx.is_defined_macro(
        make_ref_transform_iterator((*begin).children.begin(), get_value), 
        make_ref_transform_iterator((*begin).children.end(), get_value));

    ctx.enter_if_block(!is_defined);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_else(): handle #else directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_else()
{
    if (!ctx.enter_else_block()) {
    // #else without matching #if
        CPP_THROW(preprocess_exception, missing_matching_if, "#else", 
            act_pos);
    }
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_endif(): handle #endif directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_endif()
{
    if (!ctx.exit_if_block()) {
    // #endif without matching #if
        CPP_THROW(preprocess_exception, missing_matching_if, "#endif", 
            act_pos);
    }
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_if(): handle #if directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_if(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
// preprocess the given sequence into the provided list
get_token_value<result_type, parse_node_t> get_value;
std::list<result_type> expanded;
std::list<result_type> toexpand;

    std::copy(make_ref_transform_iterator(begin, get_value), 
        make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename std::list<result_type>::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded);

    std::list<result_type>::iterator exp_end = expanded.end();
    for (std::list<result_type>::iterator exp_it = expanded.begin();
         exp_it != exp_end; ++exp_it)
    {
        using namespace wave::cpplexer;
        
        token_id id = token_id(*exp_it);
        if (IS_CATEGORY(id, IdentifierTokenType) ||
            IS_CATEGORY(id, KeywordTokenType))
        {
            (*exp_it).set_token_id(T_INTLIT);
            (*exp_it).set_value("0");
        }
    }
    
#if defined(WAVE_DUMP_CONDITIONAL_EXPRESSIONS)
    {
        string_t outstr (wave::util::impl::as_string(expanded));
        WAVE_DUMP_CONDITIONAL_EXPRESSIONS_OUT << "#if " << outstr << std::endl;
    }
#endif // defined(WAVE_DUMP_CONDITIONAL_EXPRESSIONS)

// parse the expression and enter the #if block
    ctx.enter_if_block(grammars::expression_grammar_gen<result_type>::
            evaluate(expanded.begin(), expanded.end(), act_pos,
                ctx.get_if_block_status()));
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_elif(): handle #elif directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_elif(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    if (ctx.get_if_block_status()) {
        if (!ctx.enter_elif_block(false)) {
        // #else without matching #if
            CPP_THROW(preprocess_exception, missing_matching_if, "#elif", 
                act_pos);
        }
        return;     // #if/previous #elif was true, so don't enter this #elif 
    }
            
// preprocess the given sequence into the provided list
get_token_value<result_type, parse_node_t> get_value;
std::list<result_type> expanded;
std::list<result_type> toexpand;

    std::copy(make_ref_transform_iterator(begin, get_value), 
        make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename std::list<result_type>::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded);
    
    std::list<result_type>::iterator exp_end = expanded.end();
    for (std::list<result_type>::iterator exp_it = expanded.begin();
         exp_it != exp_end; ++exp_it)
    {
        using namespace wave::cpplexer;
        
        token_id id = token_id(*exp_it);
        if (IS_CATEGORY(id, IdentifierTokenType) ||
            IS_CATEGORY(id, KeywordTokenType))
        {
            (*exp_it).set_token_id(T_INTLIT);
            (*exp_it).set_value("0");
        }
    }

#if defined(WAVE_DUMP_CONDITIONAL_EXPRESSIONS)
    {
        string_t outstr (wave::util::impl::as_string(expanded));
        WAVE_DUMP_CONDITIONAL_EXPRESSIONS_OUT << "#elif " << outstr << std::endl;
    }
#endif // defined(WAVE_DUMP_CONDITIONAL_EXPRESSIONS)

// parse the expression and enter the #elif block
    ctx.enter_elif_block(grammars::expression_grammar_gen<result_type>::
            evaluate(expanded.begin(), expanded.end(), act_pos,
                ctx.get_if_block_status()));
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_illformed(): handles the illegal directive
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_illformed(
    typename result_type::string_t const &s)
{
//  skip this directive, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

    CPP_THROW(preprocess_exception, ill_formed_directive, s, act_pos);
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_line(): handle #line directives
//
///////////////////////////////////////////////////////////////////////////////

namespace {

    template <typename IteratorT, typename StringT>
    bool retrieve_line_info (IteratorT first, IteratorT const &last,
        int &line, StringT &file)
    {
        using namespace wave::cpplexer;
        if (T_INTLIT == token_id(*first)) {
        // extract line number
            using namespace std;    // some system have atoi in namespace std
            line = atoi((*first).get_value().c_str());
            
        // extract file name (if it is given)
            while (++first != last && IS_CATEGORY(*first, WhiteSpaceTokenType)) 
                /**/;   // skip whitespace
                
            if (first != last && T_STRINGLIT == token_id(*first)) {
                StringT const &file_lit = (*first).get_value();
                file = file_lit.substr(1, file_lit.size()-2);
            }
            return true;
        }
        return false;
    }
}

template <typename ContextT> 
inline void  
pp_iterator_functor<ContextT>::on_line(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
//  skip this #line, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

// Try to extract the line number and file name from the given token list
// directly. If that fails, preprocess the whole token sequence and try again 
// to extract this information.
std::list<result_type> expanded;
get_token_value<result_type, parse_node_t> get_value;

    typedef typename ref_transform_iterator_generator<
            get_token_value<result_type, parse_node_t>, 
            typename parse_tree_t::const_iterator
        >::type const_tree_iterator_t;
        
const_tree_iterator_t first = make_ref_transform_iterator(begin, get_value);
const_tree_iterator_t last = make_ref_transform_iterator(end, get_value);
    
// try to interpret the #line body as a number followed by an optional
// string literal
int line = 0;
string_t file_name;

    if (!retrieve_line_info(first, last, line, file_name)) {
    // preprocess the body of this #line message
    std::list<result_type> toexpand;

        std::copy(first, make_ref_transform_iterator(end, get_value),
            std::inserter(toexpand, toexpand.end()));

        typename std::list<result_type>::iterator begin2 = toexpand.begin();
        ctx.expand_whole_tokensequence(begin2, toexpand.end(), 
            expanded, false);
            
        if (!retrieve_line_info(expanded.begin(), expanded.end(), line, 
            file_name))
        {
            CPP_THROW(preprocess_exception, bad_line_statement, 
                wave::util::impl::as_string(expanded), act_pos)
        }
    }
    
// the queues should be empty at this point
    BOOST_SPIRIT_ASSERT(unput_queue.empty());
    BOOST_SPIRIT_ASSERT(pending_queue.empty());

    if (!file_name.empty())     // reuse current file name 
        act_pos.set_file(file_name.c_str());
    act_pos.set_line(line-1);
    last_line = act_token.get_position().get_line();
    
    must_emit_line_directive = true;
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_error(): handle #error directives
//
///////////////////////////////////////////////////////////////////////////////
namespace {

    // trim all whitespace from the begin and the end of the given string
    template <typename StringT>
    inline StringT 
    trim_whitespace(StringT const &s)
    {
        typedef typename StringT::size_type size_type;
        
        size_type first = s.find_first_not_of(" \t\v\f");
        if (StringT::npos == first)
            return StringT();
        size_type last = s.find_last_not_of(" \t\v\f");
        return s.substr(first, last-first+1);
    }
}

template <typename ContextT> 
inline void 
pp_iterator_functor<ContextT>::on_error(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
//  skip this #error, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

// preprocess the given sequence into the provided list
std::list<result_type> expanded;
get_token_value<result_type, parse_node_t> get_value;

typename ref_transform_iterator_generator<
        get_token_value<result_type, parse_node_t>, 
        typename parse_tree_t::const_iterator
    >::type first = make_ref_transform_iterator(begin, get_value);
    
#if defined(WAVE_PREPROCESS_ERROR_MESSAGE_BODY)
// preprocess the body of this #warning message
std::list<result_type> toexpand;

    std::copy(first, make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename std::list<result_type>::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded, 
        false);
#else
// simply copy the body of this #warning message to the issued diagnostic
// message
    std::copy(first, make_ref_transform_iterator(end, get_value), 
        std::inserter(expanded, expanded.end()));
#endif // defined(WAVE_PREPROCESS_ERROR_MESSAGE_BODY)

// throw the corresponding exception
    CPP_THROW(preprocess_exception, error_directive, 
        wave::util::impl::as_string(expanded), act_pos);
}

#if defined(WAVE_SUPPORT_WARNING_DIRECTIVE)
///////////////////////////////////////////////////////////////////////////////
//  
//  on_warning(): handle #warning directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void 
pp_iterator_functor<ContextT>::on_warning(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
//  skip this #warning, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

// preprocess the given sequence into the provided list
std::list<result_type> expanded;
get_token_value<result_type, parse_node_t> get_value;

typename ref_transform_iterator_generator<
        get_token_value<result_type, parse_node_t>, 
        typename parse_tree_t::const_iterator
    >::type first = make_ref_transform_iterator(begin, get_value);
    
#if defined(WAVE_PREPROCESS_ERROR_MESSAGE_BODY)
// preprocess the body of this #warning message
std::list<result_type> toexpand;

    std::copy(first, make_ref_transform_iterator(end, get_value),
        std::inserter(toexpand, toexpand.end()));

    typename std::list<result_type>::iterator begin2 = toexpand.begin();
    ctx.expand_whole_tokensequence(begin2, toexpand.end(), expanded, 
        false);
#else
// simply copy the body of this #warning message to the issued diagnostic
// message
    std::copy(first, make_ref_transform_iterator(end, get_value), 
        std::inserter(expanded, expanded.end()));
#endif // defined(WAVE_PREPROCESS_ERROR_MESSAGE_BODY)

// throw the corresponding exception
    CPP_THROW(preprocess_exception, warning_directive, 
        wave::util::impl::as_string(expanded), act_pos);
}
#endif // defined(WAVE_SUPPORT_WARNING_DIRECTIVE)

///////////////////////////////////////////////////////////////////////////////
//  
//  on_pragma(): handle #pragma directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline bool
pp_iterator_functor<ContextT>::on_pragma(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
    using namespace wave::cpplexer;
    
//  skip this #pragma, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return true;

// Look at the pragma token sequence and decide, if the first token is STDC
// (see C99 standard [6.10.6.2]), if it is, the sequence must _not_ be
// preprocessed.
std::list<result_type> expanded;
get_token_value<result_type, parse_node_t> get_value;

    typedef typename ref_transform_iterator_generator<
            get_token_value<result_type, parse_node_t>, 
            typename parse_tree_t::const_iterator
        >::type const_tree_iterator_t;
        
const_tree_iterator_t first = make_ref_transform_iterator(begin, get_value);
const_tree_iterator_t last = make_ref_transform_iterator(end, get_value);

    BOOST_SPIRIT_ASSERT(T_PP_PRAGMA == token_id(act_token));
    expanded.push_back(act_token);     // T_PP_PRAGMA;
    expanded.push_back(result_type(T_SPACE, " ", act_token.get_position()));
        
    while (++first != last && IS_CATEGORY(*first, WhiteSpaceTokenType)) 
        expanded.push_back(*first);   // skip whitespace

    if (first != last) {
        if (T_IDENTIFIER == token_id(*first) && 
            wave::need_c99(ctx.get_language()) && 
            (*first).get_value() == "STDC") 
        {
        // do _not_ preprocess the token sequence
            std::copy(first, last, std::inserter(expanded, expanded.end()));
        }
        else {
#if defined(WAVE_PREPROCESS_PRAGMA_BODY)
        // preprocess the given tokensequence
        std::list<result_type> toexpand;

            std::copy(first, last, std::inserter(toexpand, toexpand.end()));

            typename std::list<result_type>::iterator begin2 = toexpand.begin();
            ctx.expand_whole_tokensequence(begin2, toexpand.end(), 
                expanded, false);
#else
        // do _not_ preprocess the token sequence
            std::copy(first, last, std::inserter(expanded, expanded.end()));
#endif // defined(WAVE_PREPROCESS_PRAGMA_BODY)
        }
    }
    expanded.push_back(result_type(T_NEWLINE, "\n", act_token.get_position()));
        
// the queues should be empty at this point
    BOOST_SPIRIT_ASSERT(unput_queue.empty());
    BOOST_SPIRIT_ASSERT(pending_queue.empty());

// try to interpret the expanded #pragma body 
    std::list<result_type> pending;
    if (interpret_pragma(expanded, pending)) {
    // if there is some replacement text, insert it into the pending queue
        if (pending.size() > 0)
            pending_queue.splice(pending_queue.begin(), pending);
        return true;        // this #pragma was successfully recognized
    }
    
#if defined(WAVE_EMIT_PRAGMA_DIRECTIVES)
// Move the resulting token sequence into the pending_queue, so it will be 
// returned to the caller.
    pending_queue.splice(pending_queue.begin(), expanded);
    return false;           // return the whole #pragma directive
#else
    return true;            // skip the #pragma at all
#endif // defined(WAVE_EMIT_PRAGMA_DIRECTIVES)
}

template <typename ContextT> 
inline bool
pp_iterator_functor<ContextT>::interpret_pragma(
    std::list<result_type> const &pragma_body, std::list<result_type> &result)
{
    using namespace cpplexer;
    
    typename std::list<result_type>::const_iterator end = pragma_body.end();
    typename std::list<result_type>::const_iterator it = pragma_body.begin();
    for (++it; it != end && IS_CATEGORY(*it, WhiteSpaceTokenType); ++it) 
        /**/;   // skip whitespace
    
    if (it == end)      // eof reached
        return false;

    return wave::util::interpret_pragma(ctx, act_token, it, end, result,
        ctx.get_language());
}

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
///////////////////////////////////////////////////////////////////////////////
//  
//  on_region(): handle #region directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void
pp_iterator_functor<ContextT>::on_region(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
//  skip this #region, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

//  if there is a name given, start a named region
    if (begin != end) {
    // (re-)open named region
    get_token_value<result_type, parse_node_t> get_value;
    
        ctx.begin_scope(
            make_ref_transform_iterator((*begin).children.begin(), get_value),
            make_ref_transform_iterator((*begin).children.end(), get_value));
    }
    else {
    // (re-)open unnamed region
        ctx.begin_unnamed_scope();
    }
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_endregion(): handle #endregion directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void
pp_iterator_functor<ContextT>::on_endregion()
{
//  skip this #endregion, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;
    ctx.end_scope();
}

///////////////////////////////////////////////////////////////////////////////
//  
//  on_import(): handle #import directives
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT> 
inline void
pp_iterator_functor<ContextT>::on_import(
    typename parse_tree_t::const_iterator const &begin,
    typename parse_tree_t::const_iterator const &end)
{
//  skip this #import, if conditional compilation is off
    if (!ctx.get_if_block_status()) 
        return;

// import all given macro names/scope names
    get_token_value<result_type, parse_node_t> get_value;
    for (typename parse_tree_t::const_iterator it = begin; it != end; ++it) {
    // import every single of the listed names    
        ctx.import_name(
            make_ref_transform_iterator((*it).children.begin(), get_value), 
            make_ref_transform_iterator((*it).children.end(), get_value));
    }
}

#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

///////////////////////////////////////////////////////////////////////////////
// 
//  class functor_input
// 
//      Implementation of the InputPolicy used by multi_pass 
//      functor_input gets tokens from a functor
//      Note: the functor must have a typedef for result_type
//            It also must have a static variable of type result_type defined 
//            to represent eof that is called eof.
//
//      This functor input policy template is essentially the same as the 
//      predefined multi_pass functor_input policy. The only difference is, 
//      that the first token is not read at initialization time, but only 
//      just before returning the first token.
//
///////////////////////////////////////////////////////////////////////////////
struct functor_input {

    template <typename FunctorT>
    class inner {
    
        typedef typename FunctorT::result_type result_type;

    public:
        typedef result_type value_type;
        typedef std::ptrdiff_t difference_type;
        typedef result_type *pointer;
        typedef result_type &reference;

    protected:
        inner()
        :   ftor(0), curtok(0), was_initialized(false)
        {}

        inner(FunctorT const &x)
        :   ftor(new FunctorT(x)), curtok(new result_type), 
            was_initialized(false)
        {}

        inner(inner const &x)
        :   ftor(x.ftor), curtok(x.curtok)
        {}

        void destroy()
        {
            delete ftor;
            ftor = 0;
            delete curtok;
            curtok = 0;
        }

        bool same_input(inner const &x) const
        {
            return ftor == x.ftor;
        }

        void swap(inner &x)
        {
            impl::mp_swap(curtok, x.curtok);
            impl::mp_swap(ftor, x.ftor);
        }
        
        void ensure_initialized() const
        {
            if (!was_initialized && curtok) {
                *curtok = (*ftor)();    // get the first token
                was_initialized = true;
            }
        }
        
    public:
        reference get_input() const
        {
            ensure_initialized();
            return *curtok;
        }

        void advance_input()
        {
            if (curtok) {
                *curtok = (*ftor)();
            }
        }

        bool input_at_eof() const
        {
            ensure_initialized();
            return !curtok || *curtok == ftor->eof;
        }

        FunctorT& get_functor() const
        {
            return *ftor;
        }

    private:
        FunctorT *ftor;
        result_type *curtok;
        mutable bool was_initialized;
    };
};

///////////////////////////////////////////////////////////////////////////////
}   // namespace impl

///////////////////////////////////////////////////////////////////////////////
//  
//  pp_iterator
//
//      The wave::pp_iterator template is the iterator, through which
//      the resulting preprocessed input stream is accessible.
//  
///////////////////////////////////////////////////////////////////////////////

template <typename ContextT>
class pp_iterator 
:   public boost::spirit::multi_pass<
        wave::impl::pp_iterator_functor<ContextT>,
        wave::impl::functor_input
    >
{
public:
    typedef wave::impl::pp_iterator_functor<ContextT> input_policy_t;

private:
    typedef 
        boost::spirit::multi_pass<input_policy_t, wave::impl::functor_input>
        base_t;
    typedef pp_iterator<ContextT> self_t;
    typedef wave::impl::functor_input functor_input_t;
    
public:
    pp_iterator() 
    {}
    
    template <typename IteratorT>
    pp_iterator(ContextT &ctx, IteratorT const &first, IteratorT const &last,
        typename ContextT::position_t const &pos, 
        wave::language_support language)
    :   base_t(input_policy_t(ctx, first, last, pos, language))
    {}
    
    void force_include(char const *path_, bool is_last)
    { 
        get_functor().on_include_helper(path_, false); 
        if (is_last) {
            this->functor_input_t::
                template inner<input_policy_t>::advance_input();
        }
    }
};

///////////////////////////////////////////////////////////////////////////////
}   // namespace wave

#endif // !defined(CPP_ITERATOR_HPP_175CA88F_7273_43FA_9039_BCF7459E1F29_INCLUDED)

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

Share

About the Author

Hartmut Kaiser

United States United States
Actively involved in Boost and the development of the Spirit parser construction framework.

| Advertise | Privacy | Terms of Use | Mobile
Web04 | 2.8.141223.1 | Last Updated 11 Jan 2004
Article Copyright 2003 by Hartmut Kaiser
Everything else Copyright © CodeProject, 1999-2014
Layout: fixed | fluid