Click here to Skip to main content
15,861,168 members
Articles / Programming Languages / C++

Wave: a Standard conformant C++ preprocessor library

Rate me:
Please Sign up or sign in to vote.
4.96/5 (58 votes)
10 Jan 200413 min read 392.1K   4.4K   81  
Describes a free and fully Standard conformant C++ preprocessor library
/*=============================================================================
    Wave: A Standard compliant C++ preprocessor

    Macro expansion engine
    
    Copyright (c) 2001-2003 Hartmut Kaiser
    http://spirit.sourceforge.net/

    Permission to copy, use, modify, sell and distribute this software
    is granted provided this copyright notice appears in all copies.
    This software is provided "as is" without express or implied
    warranty, and with no claim as to its suitability for any purpose.

    See Copyright.txt for full copyright notices and acknowledgements.
=============================================================================*/

#if !defined(CPP_MACROMAP_HPP_CB8F51B0_A3F0_411C_AEF4_6FF631B8B414_INCLUDED)
#define CPP_MACROMAP_HPP_CB8F51B0_A3F0_411C_AEF4_6FF631B8B414_INCLUDED

#include <cstdlib>
#include <ctime>

#include <list>
#include <map>
#include <vector>
#include <iterator>
#include <algorithm>
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
#include <stack>
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

#include <boost/filesystem/path.hpp>

#include "wave/util/time_conversion_helper.hpp"
#include "wave/util/unput_queue_iterator.hpp"
#include "wave/util/macro_helpers.hpp"
#include "wave/util/macro_definition.hpp"
#include "wave/util/symbol_table.hpp"
#include "wave/grammars/cpp_defined_grammar_gen.hpp"

#include "wave/wave_version.hpp"
#include "wave/cpp_exceptions.hpp"
#include "wave/macro_trace_policies.hpp"
#include "wave/language_support.hpp"

///////////////////////////////////////////////////////////////////////////////
namespace wave {
namespace util {

namespace on_exit {

    // on destruction pop the first element of the argument list
    template <typename TokenT>
    struct pop_front {

        pop_front(std::list<TokenT> &list_) : list(list_) {}
        ~pop_front() { list.pop_front(); }
        
        std::list<TokenT> &list;
    };

    // append a given list to the argument list
    // on destruction pop the first element of the argument list
    template <typename TokenT>
    struct splice_pop_front {

        typedef std::list<TokenT> container_t;
        
        splice_pop_front(container_t &list_, container_t &queue) 
        :   list(list_) 
        {
            list.splice(list.end(), queue);
        }
        ~splice_pop_front() { list.pop_front(); }
        
        std::list<TokenT> &list;
    };

    // on destruction reset a referenced value to its initial state
    template <typename TypeT>
    struct reset {
    
        reset(TypeT &target_value_, TypeT new_value)
        :   target_value(target_value_), old_value(target_value_)
        {
            target_value_ = new_value;
        }
        ~reset() { target_value = old_value; }
        
        TypeT &target_value;
        TypeT old_value;
    };

    // on destruction assign the given iterator back
    template <typename IteratorT, typename UnputIteratorT>
    struct assign {

        assign(IteratorT &it_, UnputIteratorT const &uit_) 
        :   it(it_), uit(uit_) {}
        ~assign() { it = uit.base(); }
        
        IteratorT &it;
        UnputIteratorT const &uit;
    };

}   // namespace on_exit

///////////////////////////////////////////////////////////////////////////////
//
//  macromap
// 
//      This class holds all currently defined macros
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
class macromap {

    typedef macromap<ContextT, TraceT>              self_t;
    typedef typename ContextT::token_t              token_t;
    typedef typename token_t::string_t              string_t;
    typedef macro_definition<token_t>               macro_definition_t;
    
    typedef symbol_table<string_t, macro_definition_t>  defined_macros_t;
    typedef typename defined_macros_t::value_type::second_type macro_ref_t;

    typedef std::vector<token_t>                    parameter_container_t;
    typedef std::list<token_t>                      definition_container_t;
    
public:
    macromap(ContextT &ctx_, TraceT const &trace_) 
    :   current_macros(0), defined_macros(new defined_macros_t), 
        main_pos("", 0), ctx(ctx_), trace(trace_)
    {
        current_macros = defined_macros.get();
    }
    ~macromap() {}

    bool add_macro(token_t const &name, bool has_parameters, 
        parameter_container_t &parameters, definition_container_t &definition, 
        bool is_predefined = false, defined_macros_t *scope = 0);
    bool is_defined(string_t const &name, defined_macros_t const *scope = 0) const;
    template <typename IteratorT>
    bool is_defined(IteratorT const &begin, IteratorT const &end);
    bool remove_macro(string_t const &name, bool even_predefined = false);
    
    template <typename IteratorT, typename ContainerT>
    token_t const &expand_tokensequence(IteratorT &first, IteratorT const &last,
        ContainerT &pending, ContainerT &expanded, bool expand_undefined);

    template <typename IteratorT, typename ContainerT>
    void expand_whole_tokensequence(ContainerT &expanded, 
        IteratorT const &first, IteratorT const &last, bool expand_undefined,
        bool *seen_qualified_name = 0);

    void init_predefined_macros(defined_macros_t *scope = 0, 
        bool at_global_scope = true);
    void predefine_macro(defined_macros_t *scope, string_t const &name, 
        token_t const &t);
    void reset_macromap();

    typename token_t::position_t &get_main_pos() { return main_pos; }
    
// enable/disable tracing
    void enable_tracing(bool enable) { trace.enable_tracing(enable); }

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
// experimental: macro scoping support
    template <typename IteratorT>
    void import_name(IteratorT const &begin, IteratorT const &end);
    template <typename IteratorT>
    void begin_scope(IteratorT const &begin, IteratorT const &end);
    void begin_unnamed_scope();
    void end_scope();
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

protected:
    template <typename IteratorT, typename ContainerT>
    token_t const &expand_tokensequence_worker(ContainerT &pending, 
        unput_queue_iterator<IteratorT, token_t, ContainerT> &first, 
        unput_queue_iterator<IteratorT, token_t, ContainerT> const &last, 
        bool expand_undefined, bool *seen_qualified_name = 0);
    template <typename IteratorT, typename ContainerT>
    token_t const &expand_tokensequence_worker_classical(ContainerT &pending, 
        unput_queue_iterator<IteratorT, token_t, ContainerT> &first, 
        unput_queue_iterator<IteratorT, token_t, ContainerT> const &last, 
        bool expand_undefined);
        
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    template <typename IteratorT, typename ContainerT>
    token_t const &expand_tokensequence_worker_cpp0x(ContainerT &pending, 
        unput_queue_iterator<IteratorT, token_t, ContainerT> &first, 
        unput_queue_iterator<IteratorT, token_t, ContainerT> const &last, 
        bool expand_undefined, bool *seen_qualified_name = 0);
    void open_scope(string_t const &name, defined_macros_t *current_scope, 
        boost::shared_ptr<defined_macros_t> &new_scope);
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    template <typename IteratorT, typename ContainerT, typename SizeT>
    typename std::vector<ContainerT>::size_type collect_arguments (
        token_t const curr_token, std::vector<ContainerT> &arguments, 
        IteratorT &next, IteratorT const &end, SizeT const &parameter_count);

    template <typename IteratorT, typename ContainerT>
    bool expand_macro(ContainerT &pending, token_t const &name, 
        IteratorT &first, IteratorT const &last, bool expand_undefined,
        defined_macros_t *scope = 0, ContainerT *queue_symbol = 0,
        bool *seen_qualified_name = 0);

    template <typename ContainerT>
    bool expand_predefined_macro(token_t const &curr_token, 
        ContainerT &expanded);

    template <typename ContainerT>
    void expand_arguments (std::vector<ContainerT> &arguments, 
        std::vector<ContainerT> &expanded_args, bool expand_undefined);

    template <typename ContainerT>
    void expand_replacement_list(
        macro_definition<token_t> const &macrodefinition,
        std::vector<ContainerT> &arguments, 
        std::vector<ContainerT> const &expanded_args, ContainerT &expanded);

    template <typename ContainerT>
    void rescan_replacement_list(token_t const &curr_token, 
        macro_definition<token_t> &macrodef, ContainerT &replacement_list, 
        ContainerT &expanded, bool expand_undefined, bool *seen_qualified_name);

    template <typename IteratorT, typename ContainerT>
    token_t const &resolve_defined(IteratorT &first, IteratorT const &last,
        ContainerT &expanded);
    template <typename IteratorT, typename ContainerT>
    bool resolve_operator_pragma(IteratorT &first, 
        IteratorT const &last, ContainerT &expanded);

    template <typename ContainerT>
    void concat_tokensequence(ContainerT &expanded);

    template <typename ContainerT>
    bool is_invalid_concat(string_t new_value, 
        typename token_t::position_t const &pos, ContainerT &rescanned);

private:
    defined_macros_t *current_macros;                   // current symbol table
    boost::shared_ptr<defined_macros_t> defined_macros; // global symbol table
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    std::stack<boost::shared_ptr<defined_macros_t> > scopes;   // opened scopes
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    token_t act_token;                  // current token
    typename token_t::position_t main_pos;  // last token position in the pp_iterator
    ContextT &ctx;                      // context object associated with the macromap
    TraceT trace;                       // object which logs the macro expansion
};

///////////////////////////////////////////////////////////////////////////////
// 
//  add_macro(): adds a new macro to the macromap
//
///////////////////////////////////////////////////////////////////////////////
namespace {

    //  Test is a given identifier resolves to a predefined name
    template <typename StringT>
    inline bool 
    is_special_macroname (StringT const &name)
    {
        if (name.size() < 7)
            return false;
            
        if ("defined" == name)
            return true;
            
        if ('_' == name[0] && '_' == name[1]) {
        StringT str = name.substr(2);
        
            if (str == "cplusplus"  || str == "STDC__" || 
                str == "TIME__"     || str == "DATE__" ||
                str == "LINE__"     || str == "FILE__" ||
                str == "INCLUDE_LEVEL__"
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
                || str == "STDC_FULL_" WAVE_PP_REGION_UC "__"
                || str == "STDC_CURRENT_" WAVE_PP_REGION_UC "__"
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
                )
            {
                return true;
            }
        }
        return false;
    }
}

template <typename ContextT, typename TraceT>
inline bool 
macromap<ContextT, TraceT>::add_macro(token_t const &name, bool has_parameters, 
    parameter_container_t &parameters, definition_container_t &definition, 
    bool is_predefined, defined_macros_t *scope)
{
// exclude special macro names
    if (!is_predefined && (
            is_special_macroname (name.get_value()) || 
            IS_EXTCATEGORY(name, AltTokenType)
        ))
    {
        CPP_THROW(preprocess_exception, illegal_redefinition, 
            name.get_value(), main_pos);
    }
    
// try to define the new macro
defined_macros_t *current_scope = scope ? scope : current_macros;
typename defined_macros_t::iterator it = current_scope->find(name.get_value());

    if (it != current_scope->end()) {
    // redefinition, should not be different
#if !defined(WAVE_USE_TST_SYMBOLTABLE)
        if ((*it).second->is_functionlike != has_parameters ||
            (*it).second->macroparameters != parameters ||
            (*it).second->macrodefinition != definition)
#else
        if ((*it).data().is_functionlike != has_parameters ||
            (*it).data().macroparameters != parameters ||
            (*it).data().macrodefinition != definition)
#endif // !defined(WAVE_USE_TST_SYMBOLTABLE)
        {
            CPP_THROW(preprocess_exception, macro_redefinition, 
                name.get_value(), main_pos);
        }
        return false;
    }

// test the validity of the parameter names
    if (has_parameters) {
        std::set<typename token_t::string_t> names;
    
        typedef typename parameter_container_t::iterator 
            parameter_iterator_t;
        typedef typename std::set<typename token_t::string_t>::iterator 
            name_iterator_t;
            
        parameter_iterator_t end = parameters.end();
        for (parameter_iterator_t it = parameters.begin(); it != end; ++it) {
        name_iterator_t pit = names.find((*it).get_value());
        
            if (pit != names.end()) {
            // duplicate parameter name
                CPP_THROW(preprocess_exception, duplicate_parameter_name, 
                    (*pit), main_pos);
            }
            names.insert((*it).get_value());
        }
    }
    
// insert a new macro node
#if !defined(WAVE_USE_TST_SYMBOLTABLE)    
    std::pair<typename defined_macros_t::iterator, bool> p = 
        current_scope->insert(
            defined_macros_t::value_type(
                name.get_value(), 
                macro_ref_t(new macro_definition<token_t>(name, 
                    has_parameters, is_predefined)
                )
            )
        );

    if (!p.second) {
        CPP_THROW(preprocess_exception, macro_insertion_error, 
            name.get_value(), main_pos);
    }

// add the parameters and the definition
    std::swap((*p.first).second->macroparameters, parameters);
    std::swap((*p.first).second->macrodefinition, definition);
#else
    std::pair<typename defined_macros_t::iterator, bool> p = 
        current_scope->insert(name.get_value());

    if (!p.second) {
        CPP_THROW(preprocess_exception, macro_insertion_error, 
            name.get_value(), main_pos);
    }
    (*p.first).data() = macro_definition<token_t>(name, has_parameters, 
        is_predefined);

// add the parameters and the definition
    std::swap((*p.first).data().macroparameters, parameters);
    std::swap((*p.first).data().macrodefinition, definition);
#endif // !defined(WAVE_USE_TST_SYMBOLTABLE)
    return true;
}

///////////////////////////////////////////////////////////////////////////////
// 
//  is_defined(): returnes, whether a given macro is already defined
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
inline bool 
macromap<ContextT, TraceT>::is_defined(typename token_t::string_t const &name,
    defined_macros_t const *scope) const
{
    if (0 == scope) scope = current_macros;
    
    return scope->find(name) != scope->end() ||
        name == "__LINE__" || name == "__FILE__" || 
        name == "__INCLUDE_LEVEL__";
}

template <typename ContextT, typename TraceT>
template <typename IteratorT>
inline bool 
macromap<ContextT, TraceT>::is_defined(IteratorT const &begin, 
    IteratorT const &end) 
{
    using namespace cpplexer;
    
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    if (wave::need_cpp0x(ctx.get_language())) {
    defined_macros_t *current_scope = 0;
    bool was_colon_colon = false;

        for (IteratorT it = begin; it != end; /**/) {
            if (token_id(*it) == T_COLON_COLON) {
                if (was_colon_colon) {
                // error: two consecutive '::'
                    CPP_THROW(preprocess_exception, invalid_macroname, 
                        get_full_name(begin, end), main_pos);
                }
                if (0 == current_scope) {
                // initialize starting scope (global scope)
                    current_scope = defined_macros.get();
                }
                was_colon_colon = true;
                ++it;
            }
            else if (T_IDENTIFIER == token_id(*it) ||
                IS_CATEGORY(*it, KeywordTokenType))
            {
                if (0 == current_scope) {
                // initialize starting scope
                    current_scope = current_macros;     // start at current scope
                }
                else if (!was_colon_colon) {
                // error: missing '::' between two parts of the macro name
                    CPP_THROW(preprocess_exception, invalid_macroname, 
                        get_full_name(begin, end), main_pos);
                }
                was_colon_colon = false;
                
            string_t name ((*it).get_value());
            boost::shared_ptr<defined_macros_t> next_scope;
            typename defined_macros_t::const_iterator cit = 
                current_scope -> find(name);
            
                if (cit != current_scope -> end()) {
                // defined as a macro name
                
                // should be the last given name part 
                    if (++it != end) 
                        break;      // consider as not defined

                // import this macro name
                    return true;    // defined as a macro
                }
                else if (current_scope -> get_scope(name, next_scope)) {
                // if this part was the last one, then import the whole scope
                    if (++it != end) {
                    // defined as a scope name
                        current_scope = next_scope.get();
                    }  
                    else {
                        break;      // not found
                    }              
                }
                else {
                    break;          // name is not defined
                }
            }
            else {
                if (!IS_CATEGORY(*it, WhiteSpaceTokenType)) {
                // invalid token type inside a qualified name
                    CPP_THROW(preprocess_exception, invalid_macroname, 
                        get_full_name(begin, end), main_pos);
                }
                ++it;
            }
        }
    }
    else 
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    {
    // in normal mode the name under inspection should consist of an identifer
    // only
        if (T_IDENTIFIER != token_id(*begin) && 
            !IS_CATEGORY(*begin, KeywordTokenType)) 
        {
            if (is_qualified_name(begin, end)) {
                CPP_THROW(preprocess_exception, unexpected_qualified_name, 
                    get_full_name(begin, end), main_pos);
            }
            else {
                CPP_THROW(preprocess_exception, invalid_macroname, 
                    get_full_name(begin, end), main_pos);
            }
        }

    IteratorT it = begin;
    string_t name ((*it).get_value());
    typename defined_macros_t::const_iterator cit = 
        current_macros -> find(name);

        if (++it != end) {
        // there should be only one token as the inspected name
            if (is_qualified_name(begin, end)) {
                CPP_THROW(preprocess_exception, unexpected_qualified_name, 
                    get_full_name(begin, end), main_pos);
            }
            else {
                CPP_THROW(preprocess_exception, invalid_macroname, 
                    get_full_name(begin, end), main_pos);
            }
        }
        return cit != current_macros -> end();
    }
    return false;       // not defined
}

///////////////////////////////////////////////////////////////////////////////
// 
//  remove_macro(): remove a macro from the macromap
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
inline bool 
macromap<ContextT, TraceT>::remove_macro(string_t const &name, 
    bool even_predefined)
{
    typename defined_macros_t::iterator it = current_macros->find(name);
    
    if (it != current_macros->end()) {
#if !defined(WAVE_USE_TST_SYMBOLTABLE)
        if ((*it).second->is_predefined) {
#else
        if ((*it).data().is_predefined) {
#endif // !defined(WAVE_USE_TST_SYMBOLTABLE)
            if (!even_predefined || is_special_macroname(name)) {
                CPP_THROW(preprocess_exception, bad_undefine_statement, 
                    name, main_pos);
            }
        }
        current_macros->erase(it);
        return true;
    }
    else if (is_special_macroname(name)) {
        CPP_THROW(preprocess_exception, bad_undefine_statement, 
            name, main_pos);
    }
    return false;       // macro was not defined
}

///////////////////////////////////////////////////////////////////////////////
// 
//  expand_tokensequence
//
//      This function is a helper function which wraps the given iterator 
//      range into corresponding unput_iterator's and calls the main workhorse
//      of the macro expansion engine (the function expand_tokensequence_worker)
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT>
inline typename ContextT::token_t const &
macromap<ContextT, TraceT>::expand_tokensequence(IteratorT &first, 
    IteratorT const &last, ContainerT &pending, ContainerT &expanded, 
    bool expand_undefined)
{
    typedef impl::gen_unput_queue_iterator<IteratorT, token_t, ContainerT> gen_t;
    typedef typename gen_t::return_t iterator_t;

    ContainerT eof_queue;
    iterator_t first_it = gen_t::generate(expanded, first);
    iterator_t last_it = gen_t::generate(eof_queue, last);

bool seen_qualified_name = false;
on_exit::assign<IteratorT, iterator_t> on_exit(first, first_it);

    return expand_tokensequence_worker(pending, first_it, last_it, 
        expand_undefined, &seen_qualified_name);
}

///////////////////////////////////////////////////////////////////////////////
//
//  expand_tokensequence_worker
//
//      This function is the main workhorse of the macro expansion engine. It
//      expands as much tokens as needed to identify the next preprocessed 
//      token to return to the caller. 
//      It returns the next preprocessed token.
//
//      The iterator 'first' is adjusted accordingly.
//
///////////////////////////////////////////////////////////////////////////////

namespace {

    // strip leading and trailing whitespace
    template <typename ContainerT>
    inline void
    trim_replacement_list (ContainerT &replacement_list)
    {
        using namespace wave::cpplexer;

    // strip leading whitespace
        if (replacement_list.size() > 0) {
        typename ContainerT::iterator end = replacement_list.end();
        typename ContainerT::iterator it = replacement_list.begin();
        
            while (it != end && IS_CATEGORY(*it, WhiteSpaceTokenType)) { 
                if (T_PLACEHOLDER != token_id(*it)) {
                    typename ContainerT::iterator next = it;
                    ++next;
                    replacement_list.erase(it);
                    it = next;
                }
                else {
                    ++it;
                }
            }
        }
        
    // strip trailing whitespace
        if (replacement_list.size() > 0) {
        typename ContainerT::reverse_iterator rend = replacement_list.rend();
        typename ContainerT::reverse_iterator rit = replacement_list.rbegin();
        
            while (rit != rend && IS_CATEGORY(*rit, WhiteSpaceTokenType)) 
                ++rit;

        typename ContainerT::iterator end = replacement_list.end();
        typename ContainerT::iterator it = rit.base();
        
            while (it != end && IS_CATEGORY(*it, WhiteSpaceTokenType)) { 
                if (T_PLACEHOLDER != token_id(*it)) {
                    typename ContainerT::iterator next = it;
                    ++next;
                    replacement_list.erase(it);
                    it = next;
                }
                else {
                    ++it;
                }
            }
        }
    }

    // remove all placeholder tokens from the given token sequence
    template <typename ContainerT>
    inline void
    remove_placeholders (ContainerT &replacement_list)
    {
        using namespace wave::cpplexer;

    // strip leading whitespace
        if (replacement_list.size() > 0) {
        typename ContainerT::iterator end = replacement_list.end();
        typename ContainerT::iterator it = replacement_list.begin();
        
            while (it != end) {
                if (T_PLACEHOLDER == token_id(*it)) {
                    typename ContainerT::iterator next = it;
                    ++next;
                    replacement_list.erase(it);
                    it = next;
                }
                else {
                    ++it;
                }
            }
            
        // remove all 'new' leading and trailing whitespace 
            trim_replacement_list(replacement_list);
        }
    }
}

template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT>
inline typename ContextT::token_t const &
macromap<ContextT, TraceT>::expand_tokensequence_worker(
    ContainerT &pending, 
    unput_queue_iterator<IteratorT, token_t, ContainerT> &first, 
    unput_queue_iterator<IteratorT, token_t, ContainerT> const &last, 
    bool expand_undefined, bool *seen_qualified_name)
{
// if there exist pending tokens (tokens, which are already preprocessed), then
// return the next one from there
    if (!pending.empty()) {
    on_exit::pop_front<token_t> pop_front_token(pending);
    
        return act_token = pending.front();
    }

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    if (wave::need_cpp0x(ctx.get_language()))
        return expand_tokensequence_worker_cpp0x(pending, first, last, 
            expand_undefined, seen_qualified_name);
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

    return expand_tokensequence_worker_classical(pending, first, last, 
        expand_undefined);
}

template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT>
inline typename ContextT::token_t const &
macromap<ContextT, TraceT>::expand_tokensequence_worker_classical(
    ContainerT &pending, 
    unput_queue_iterator<IteratorT, token_t, ContainerT> &first, 
    unput_queue_iterator<IteratorT, token_t, ContainerT> const &last, 
    bool expand_undefined)
{
    using namespace wave::cpplexer;
    typedef unput_queue_iterator<IteratorT, token_t, ContainerT> iterator_t;
    
//  analyze the next element of the given sequence, if it is an 
//  T_IDENTIFIER token, try to replace this as a macro etc.
    if (first != last) {
    token_id id = token_id(*first);

    // ignore placeholder tokens
        if (T_PLACEHOLDER == id) {
        token_t placeholder = *first;
        
            ++first;
            if (first == last)
                return act_token = placeholder;
            id = token_id(*first);
        }
            
        if (T_IDENTIFIER == id || IS_CATEGORY(id, KeywordTokenType)) {
        // try to replace this identifier as a macro
            if (expand_undefined && (*first).get_value() == "defined") {
            // resolve operator defined()
                return resolve_defined(first, last, pending);
            }
            else if (wave::need_variadics(ctx.get_language()) && 
                (*first).get_value() == "_Pragma") 
            {
            // in C99 mode only: resolve the operator _Pragma
            token_t curr_token = *first;
            
                if (!resolve_operator_pragma(first, last, pending) ||
                    pending.size() > 0) 
                {
                // unknown to us pragma or supplied replacement, return the 
                // next token
                on_exit::pop_front<token_t> pop_front_token(pending);

                    return act_token = pending.front();
                }
                
            // the operator _Pragma() was eaten completely, continue
                return act_token = token_t(T_PLACEHOLDER, "_", 
                    curr_token.get_position());
            }

        token_t name_token (*first);
        
            if (is_defined(name_token.get_value())) {
            // the current token contains an identifier, which is currently 
            // defined as a macro
                if (expand_macro(pending, name_token, first, last, 
                    expand_undefined)) 
                {
                // the tokens returned by expand_macro should be rescanned
                // beginning at the last token of the returned replacement list
                    if (first != last) {
                    // splice the last token back into the input queue
                    typename ContainerT::reverse_iterator rit = pending.rbegin();
                    
                        first.get_unput_queue().splice(
                            first.get_unput_queue().begin(), pending,
                            (++rit).base(), pending.end());
                    }
                                            
                // fall through ...
                }
                else if (!pending.empty()) {
                // return the first token from the pending queue
                on_exit::pop_front<token_t> pop_front_queue (pending);
                
                    return act_token = pending.front();
                }
                else {
                // macro expansion reached the eoi
                    return act_token = token_t();
                }

            // return the next preprocessed token
                return expand_tokensequence_worker(pending, first, last, 
                    expand_undefined);
            }
            else if (expand_undefined) {
            // in preprocessing conditionals undefined identifiers and keywords 
            // are to be replaced with '0' (see. C++ standard 16.1.4, [cpp.cond])
                return act_token = 
                    token_t(T_INTLIT, "0", (*first++).get_position());
            }
            else {
                act_token = name_token;
                ++first;
                return act_token;
            }
        }
        else if (expand_undefined && IS_CATEGORY(*first, BoolLiteralTokenType)) {
        // expanding a constant expression inside #if/#elif, special handling
        // of 'true' and 'false'        

        // all remaining identifiers and keywords, except for true and false, 
        // are replaced with the pp-number 0 (C++ standard 16.1.4, [cpp.cond])
            return act_token = token_t(T_INTLIT, T_TRUE != id ? "0" : "1", 
                (*first++).get_position());
        }
        else {
            act_token = *first;
            ++first;
            return act_token;
        }
    }
    return act_token = token_t();     // eoi
}

///////////////////////////////////////////////////////////////////////////////
//  version to be used while in C++0x mode
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

namespace {

    ///////////////////////////////////////////////////////////////////////////
    //  skip all tokens, which relate with the current qualified name, start
    //  with the '::' token
    template <typename IteratorT, typename ContainerT>
    inline void
    skip_remaining_identifier(IteratorT &first, 
        IteratorT const &last, ContainerT &queue_symbol)
    {
        using namespace cpplexer;
        
        do {
        // skip whitespace
        token_id id = impl::next_token<IteratorT>::peek(first, last);
        
            if (T_COLON_COLON != id) 
                break;
            impl::skip_whitespace(first, last, queue_symbol);
            
        // skip whitespace again   
            id = impl::next_token<IteratorT>::peek(first, last);
            if (T_IDENTIFIER != id && 
                !IS_CATEGORY(id, KeywordTokenType))
            {
                break;
            }
            id = impl::skip_whitespace(first, last, queue_symbol);

        } while (true);
    }

    ///////////////////////////////////////////////////////////////////////////
    //  skip all tokens back, which relate with the current qualified name
    //  (the iterators are reverse_iterators)
    template <typename IteratorT>
    inline void
    rewind_identifier(IteratorT &first, IteratorT const &last)
    {
        using namespace cpplexer;
        
    IteratorT save(first);

        if (T_COLON_COLON == token_id(*save)) {
            do {
            // skip whitespace
            token_id id = impl::skip_whitespace(save, last);

                if (T_IDENTIFIER != id && 
                    !IS_CATEGORY(id, KeywordTokenType)) 
                {
                    break;
                }
            
            // save this iterator position as the new starting point
                first = save;

            // skip whitespace again   
                id = impl::skip_whitespace(save, last);
                if (T_COLON_COLON != id)
                    break;

            // save this iterator position as the new starting point
                first = save;
            
            } while (true);
        }
        else {
            do {
            // skip whitespace
            token_id id = impl::skip_whitespace(save, last);

                if (T_COLON_COLON != id) 
                    break;
                
            // save this iterator position as the new starting point
                first = save;

            // skip whitespace again   
                id = impl::skip_whitespace(save, last);
                if (T_IDENTIFIER != id && 
                    !IS_CATEGORY(id, KeywordTokenType))
                {
                    break;
                }

            // save this iterator position as the new starting point
                first = save;

            } while (true);
        }
    }
}

template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT>
inline typename ContextT::token_t const &
macromap<ContextT, TraceT>::expand_tokensequence_worker_cpp0x(
    ContainerT &pending, 
    unput_queue_iterator<IteratorT, token_t, ContainerT> &first, 
    unput_queue_iterator<IteratorT, token_t, ContainerT> const &last, 
    bool expand_undefined, bool *seen_qualified_name)
{
    using namespace wave::cpplexer;
    typedef unput_queue_iterator<IteratorT, token_t, ContainerT> iterator_t;
    
//  analyze the next element(s) of the given sequence, if it is an 
//  valid macro name, try to replace this as a macro etc.
defined_macros_t *scope = 0;
ContainerT queue_symbol;    // queue up tokens to backup the not found case
bool was_colon_colon = false;
bool found_scope = false;

    while (first != last) {
    token_id id = token_id(*first);

        if (T_COLON_COLON == id) {
            if (0 == scope) 
                scope = defined_macros.get();   // start at global scope

            if (seen_qualified_name)
                *seen_qualified_name = true;

        // continue with the next non-whitespace token
            impl::skip_whitespace(first, last, queue_symbol);
            was_colon_colon = true;
            continue;
        }

    // ignore placeholder tokens
        if (T_PLACEHOLDER == id) {
        token_t placeholder = *first;
        
            ++first;
            if (first == last)
                return act_token = placeholder;
            id = token_id(*first);
            queue_symbol.push_back(placeholder);
        }

        if (T_IDENTIFIER == id || IS_CATEGORY(id, KeywordTokenType)) {
        // try to replace this identifier as a macro
            if (expand_undefined && (*first).get_value() == "defined") {
            // resolve operator defined()
                if (0 != scope) {
                // misplaced operator defined()
                    CPP_THROW(preprocess_exception, misplaced_operator, 
                        "defined", main_pos);
                }
                if (seen_qualified_name)
                    *seen_qualified_name = false;
                return resolve_defined(first, last, pending);
            }
            else if (wave::need_variadics(ctx.get_language()) && 
                (*first).get_value() == "_Pragma") 
            {
                if (0 != scope) {
                // misplaced operator defined()
                    CPP_THROW(preprocess_exception, misplaced_operator, 
                        "_Pragma", main_pos);
                }

                if (seen_qualified_name)
                    *seen_qualified_name = false;

            // in C99 mode only: resolve the operator _Pragma
            token_t curr_token = *first;
            
                if (!resolve_operator_pragma(first, last, pending) ||
                    pending.size() > 0) 
                {
                // unknown to us pragma or supplied replacement, return the 
                // next token
                on_exit::pop_front<token_t> pop_front_token(pending);

                    return act_token = pending.front();
                }
                
            // the operator _Pragma() was eaten completely, continue
                return act_token = token_t(T_PLACEHOLDER, "_", 
                    curr_token.get_position());
            }

        // initialize scope
            if (0 == scope) {
                scope = current_macros;
            }
            else if (!was_colon_colon) {
            // not resolvable as a macro
                on_exit::splice_pop_front<token_t> 
                pop_front_queue (pending, queue_symbol);
            
                return act_token = pending.front();
            }

        // search the identifier first as a scope, and if not found, 
        // subsequently as a macro
        token_t name_token(*first);
        boost::shared_ptr<defined_macros_t> next_scope;
        
            if (scope -> get_scope(name_token.get_value(), next_scope)) {
            // continue with the next non-whitespace token
            token_id idnext = impl::skip_whitespace(first, last, queue_symbol);
            
                if (T_COLON_COLON != idnext) {
                // the name under inspection may not be a qualified macro name
                    if (seen_qualified_name)
                        *seen_qualified_name = false;

                // return the first token from the pending queue and restart 
                // with the next token
                on_exit::splice_pop_front<token_t> 
                    pop_front_queue (pending, queue_symbol);
                
                    return act_token = pending.front();
                }

            // this identifier is known here as a subscope
                scope = next_scope.get();
                was_colon_colon = false;
                found_scope = true;
                if (seen_qualified_name)
                    *seen_qualified_name = true;
                
                continue;
            }

            if (is_defined(name_token.get_value(), scope)) {
            // the current token contains an identifier, which is currently 
            // defined as a macro
            
                if (expand_macro(pending, name_token, first, last, 
                    expand_undefined, scope, &queue_symbol, seen_qualified_name)) 
                {
                // the tokens returned by expand_macro should be rescanned
                // beginning at the last token of the returned replacement list
                    if (first != last) {
                    // splice the last token back into the input queue
                    typename ContainerT::reverse_iterator rit = pending.rbegin();

                    // skip all remaining tokens _back_ if the last token is 
                    // the end of a qualified name
                        if (seen_qualified_name && *seen_qualified_name &&
                            (T_COLON_COLON == token_id(*rit) ||
                             IS_CATEGORY(*rit, IdentifierTokenType)) 
                           )
                        {
                            rewind_identifier(rit, pending.rend());
                        }
                        first.get_unput_queue().splice(
                            first.get_unput_queue().begin(), pending,
                            (++rit).base(), pending.end());
                    }
                                            
                // fall through ...
                }
                else if (!pending.empty()) {
                // there was some replacement, but no rescanning is required
                // (predefined macros, not-expandable macros etc.)

                // return the first token from the pending queue
                on_exit::splice_pop_front<token_t> 
                    pop_front_queue (pending, queue_symbol);
                
                    return act_token = pending.front();
                }
                else {
                // macro expansion reached the eoi
                    return act_token = token_t();
                }

            // return the next preprocessed token
                return expand_tokensequence_worker(pending, first, last, 
                    expand_undefined, seen_qualified_name);
            }
            else if (expand_undefined) {
            // in preprocessing conditionals undefined identifiers and keywords 
            // are to be replaced with '0' (see. C++ standard 16.1.4, [cpp.cond])
                return act_token = 
                    token_t(T_INTLIT, "0", (*first++).get_position());
            }
            else {
            // the token sequence is not known as a (qualified) macro name, so
            // make this sequence pending and return the first token
                if (seen_qualified_name)
                    *seen_qualified_name = false;

                if (found_scope) {
                // If a qualified name does not find a nested name, it is 
                // not a qualified name to the preprocessor.
                    BOOST_SPIRIT_ASSERT(queue_symbol.size() >= 2);
                
                // i.e. we have to re-evaluate the last (not found nested) 
                // name in the global context, for this reason the  
                // preceeding '::' is pushed back into the input stream
                typename ContainerT::reverse_iterator rit = queue_symbol.rbegin();

                    if (T_COLON_COLON != token_id(*rit))
                        impl::skip_whitespace(rit, queue_symbol.rend());
                    first.get_unput_queue().splice(
                        first.get_unput_queue().begin(), queue_symbol,
                        (++rit).base(), queue_symbol.end());

                // leave all up to the last '::'' in the symbol queue, which
                // is treated as preprocessed
                on_exit::splice_pop_front<token_t> 
                    pop_front_queue (pending, queue_symbol);

                    return act_token = pending.front();
                }
                    
                if (was_colon_colon) {
                // the symbol queue may only be not empty if there was a '::' 
                // followed eventually by some whitespace
                    BOOST_SPIRIT_ASSERT(!queue_symbol.empty());

                on_exit::splice_pop_front<token_t> 
                    pop_front_queue (pending, queue_symbol);

                    return act_token = pending.front();
                }

                act_token = name_token;
                ++first;
                return act_token;
            }
        }
        else if (expand_undefined && IS_CATEGORY(*first, BoolLiteralTokenType)) {
        // expanding a constant expression inside #if/#elif, special handling
        // of 'true' and 'false'        

        // all remaining identifiers and keywords, except for true and false, 
        // are replaced with the pp-number 0 (C++ standard 16.1.4, [cpp.cond])
            return act_token = token_t(T_INTLIT, T_TRUE != id ? "0" : "1", 
                (*first++).get_position());
        }
        else {
        // something else (not a T_IDENTIFIER token)
            if (seen_qualified_name && !IS_CATEGORY(*first, WhiteSpaceTokenType))
                *seen_qualified_name = false;
                
            queue_symbol.push_back(*first);
            
        on_exit::splice_pop_front<token_t> 
            pop_front_queue (pending, queue_symbol);

            ++first;
            return act_token = pending.front();
        }
    }

// end of input reached
    if (!queue_symbol.empty()) {
    on_exit::splice_pop_front<token_t> pop_front_queue (pending, queue_symbol);

        return act_token = pending.front();
    }
    return act_token = token_t();     // eoi
}
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

///////////////////////////////////////////////////////////////////////////////
// 
//  collect_arguments(): collect the actual arguments of a macro invocation
//
//      return the number of successfully detected non-empty arguments
//
///////////////////////////////////////////////////////////////////////////////

namespace {

    template <typename ContainerT>
    inline void
    trim_argument_left (ContainerT &argument)
    {
        using namespace wave::cpplexer;
        
    // strip leading whitespace (should be only one token)
        if (argument.size() > 0 &&
            IS_CATEGORY(argument.front(), WhiteSpaceTokenType))
        {
            argument.pop_front();
        }
    }
    
    template <typename ContainerT>
    inline void
    trim_argument_right (ContainerT &argument)
    {
        using namespace wave::cpplexer;
        
    // strip trailing whitespace (should be only one token)
        if (argument.size() > 0 &&
            IS_CATEGORY(argument.back(), WhiteSpaceTokenType))
        {
            argument.pop_back();
        }
    }

    template <typename ContainerT>
    inline void
    trim_argument (ContainerT &argument)
    {
        trim_argument_left(argument);
        trim_argument_right(argument);
    }
    
    template <typename ContainerT>
    inline bool
    is_whitespace_only (ContainerT const &argument)
    {
        using namespace cpplexer;
        
        typename ContainerT::const_iterator end = argument.end();
        for (typename ContainerT::const_iterator it = argument.begin();
             it != end; ++it)
        {
            if (!IS_CATEGORY(*it, WhiteSpaceTokenType))
                return false;
        }
        return true;
    }
}

template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT, typename SizeT>
inline typename std::vector<ContainerT>::size_type 
macromap<ContextT, TraceT>::collect_arguments (token_t const curr_token, 
    std::vector<ContainerT> &arguments, IteratorT &next, IteratorT const &end, 
    SizeT const &parameter_count)
{
    using namespace wave::cpplexer;
    
    arguments.push_back(ContainerT());
    
// collect the actual arguments
typename std::vector<ContainerT>::size_type count_arguments = 0;
int nested_parenthesis_level = 1;
ContainerT *argument = &arguments[0];
bool was_whitespace = false;
token_t startof_argument_list = *next;

    while (++next != end && nested_parenthesis_level) {
    token_id id = token_id(*next);

        if (0 == parameter_count && 
            !IS_CATEGORY((*next), WhiteSpaceTokenType) && id != T_NEWLINE &&
            id != T_RIGHTPAREN && id != T_LEFTPAREN) 
        {
        // there shouldn't be any arguments
            CPP_THROW(preprocess_exception, too_many_macroarguments, 
                curr_token.get_value(), main_pos);
        }
        
        switch (id) {
        case T_LEFTPAREN:
            ++nested_parenthesis_level;
            argument->push_back(*next);
            was_whitespace = false;
            break;
            
        case T_RIGHTPAREN:
            {
                if (--nested_parenthesis_level >= 1)
                    argument->push_back(*next);
                else {
                // found closing parenthesis
//                    trim_argument(argument);
                    if (parameter_count > 0) {
                        if (0 == argument->size() || 
                            is_whitespace_only(*argument)) 
                        {
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                            if (wave::need_variadics(ctx.get_language())) {
                            // store a placemarker as the argument
                                argument->push_back(token_t(T_PLACEMARKER, "�", 
                                    (*next).get_position()));
                                ++count_arguments;
                            }
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                        }
                        else {
                            ++count_arguments;
                        }
                    }
                }
                was_whitespace = false;
            }
            break;
        
        case T_COMMA:
            if (1 == nested_parenthesis_level) {
            // next parameter
//                trim_argument(argument);
                if (0 == argument->size() || 
                    is_whitespace_only(*argument)) 
                {
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                    if (wave::need_variadics(ctx.get_language())) {
                    // store a placemarker as the argument
                        argument->push_back(token_t(T_PLACEMARKER, "�", 
                            (*next).get_position()));
                        ++count_arguments;
                    }
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                }
                else {
                    ++count_arguments;
                }
                arguments.push_back(ContainerT()); // add new arg
                argument = &arguments[arguments.size()-1];
            }
            else {
            // surrounded by parenthesises, so store to current argument
                argument->push_back(*next);
            }
            was_whitespace = false;
            break;

        case T_SPACE:
        case T_SPACE2:
        case T_CCOMMENT:
        case T_NEWLINE:
            if (!was_whitespace) 
                argument->push_back(token_t(T_SPACE, " ", (*next).get_position()));
            was_whitespace = true;
            break;      // skip whitespace

        case T_PLACEHOLDER:
            break;      // ignore placeholder
                        
        default:
            argument->push_back(*next);
            was_whitespace = false;
            break;
        }
    }

    if (nested_parenthesis_level >= 1) {
    // missing ')': improperly terminated macro invocation
        CPP_THROW(preprocess_exception, improperly_terminated_macro, 
            "missing ')'", main_pos);
    }

// if there isn't expected any argument and there really wasn't found any, 
// than remove the empty element
    if (0 == parameter_count && 0 == count_arguments) {
        BOOST_SPIRIT_ASSERT(1 == arguments.size());
        arguments.clear();
    }
    return count_arguments;
}        

///////////////////////////////////////////////////////////////////////////////
// 
//  expand_macro(): expands a defined macro
//
//      This functions tries to expand the macro, to which points the 'first'
//      iterator. The functions eats up more tokens, if the macro to expand is
//      a function-like macro.
//
///////////////////////////////////////////////////////////////////////////////
namespace {

    using namespace wave::cpplexer;
    
    template <typename IteratorT>
    bool skip_to_token(IteratorT &it, IteratorT const &end, token_id id)
    {
        using namespace wave::cpplexer;
        if (token_id(*it) == id) 
            return true;
        if (++it == end) 
            return false;
        while (IS_CATEGORY(*it, WhiteSpaceTokenType) || 
               T_NEWLINE == token_id(*it)) 
        {
            if (++it == end)
                return false;
        }
        BOOST_SPIRIT_ASSERT(token_id(*it) == id);
        return true;
    }

}

///////////////////////////////////////////////////////////////////////////////
// 
//  expand_whole_tokensequence
//
//      fully expands a given token sequence 
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT>
inline void
macromap<ContextT, TraceT>::expand_whole_tokensequence(ContainerT &expanded, 
    IteratorT const &first, IteratorT const &last, bool expand_undefined,
    bool *seen_qualified_name)
{
    typedef impl::gen_unput_queue_iterator<IteratorT, token_t, ContainerT> gen_t;
    typedef typename gen_t::return_t iterator_t;

    ContainerT unput_queue;
    iterator_t first_it = gen_t::generate(unput_queue, first);
    ContainerT eof_queue;
    iterator_t last_it = gen_t::generate(eof_queue, last);

    bool was_whitespace = false;
    ContainerT pending_queue;
    while (!pending_queue.empty() || first_it != last_it) {
    token_t t = expand_tokensequence_worker(pending_queue, first_it, 
                    last_it, expand_undefined, seen_qualified_name);
    bool is_whitespace = IS_CATEGORY(t, WhiteSpaceTokenType) &&
        T_PLACEHOLDER != token_id(t);

        if (!was_whitespace || !is_whitespace) {
            if (is_whitespace && T_SPACE != token_id(t)) {
                t.set_token_id(T_SPACE);
                t.set_value(" ");
            }
            expanded.push_back(t);
        }
        was_whitespace = is_whitespace;
    }

// should have returnd all expanded tokens
    BOOST_SPIRIT_ASSERT(unput_queue.empty() && pending_queue.empty());
}

///////////////////////////////////////////////////////////////////////////////
// 
//  expand_arguments
//
//      fully expands the arguments of a macro call 
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
template <typename ContainerT>
inline void 
macromap<ContextT, TraceT>::expand_arguments (std::vector<ContainerT> &arguments, 
    std::vector<ContainerT> &expanded_args, bool expand_undefined)
{
typename std::vector<ContainerT>::size_type i = 0;

    typename std::vector<ContainerT>::iterator arg_end = arguments.end();
    for (typename std::vector<ContainerT>::iterator arg_it = arguments.begin(); 
         arg_it != arg_end; ++arg_it, ++i)
    {
    bool seen_qualified_name = false;
    
        expand_whole_tokensequence(expanded_args[i], (*arg_it).begin(), 
            (*arg_it).end(), expand_undefined, &seen_qualified_name);
        remove_placeholders(expanded_args[i]);
    }
}
                    
///////////////////////////////////////////////////////////////////////////////
// 
//  expand_replacement_list
//
//      fully expands the replacement list of a given macro with the 
//      actual arguments/expanded arguments
//      handles the '#' [cpp.stringize] and the '##' [cpp.concat] operator
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
template <typename ContainerT>
inline void
macromap<ContextT, TraceT>::expand_replacement_list(
    macro_definition<token_t> const &macrodef,
    std::vector<ContainerT> &arguments, 
    std::vector<ContainerT> const &expanded_args, ContainerT &expanded)
{
    BOOST_SPIRIT_ASSERT(arguments.size() == expanded_args.size());
    
    using namespace wave::cpplexer;
    typedef typename macro_definition_t::const_definition_iterator_t 
        macro_definition_iter_t;

bool seen_concat = false;
bool adjacent_concat = false;
bool adjacent_stringize = false;

    macro_definition_iter_t cend = macrodef.macrodefinition.end();
    for (macro_definition_iter_t cit = macrodef.macrodefinition.begin();
        cit != cend; ++cit)
    {
    bool use_replaced_arg = true;
    token_id id = token_id(*cit);
    
        if (T_POUND_POUND == id) {
        // conactination operator
            adjacent_concat = true;
            seen_concat = true;
        }
        else if (T_POUND == id) {
        // stringize operator
            adjacent_stringize = true;
            seen_concat = true;
        }
        else {
            if (adjacent_stringize || adjacent_concat || 
                T_POUND_POUND == impl::next_token<macro_definition_iter_t>
                    ::peek(cit, cend))
            {
                use_replaced_arg = false;
            }
            if (adjacent_concat)    // spaces after '##' ?
                adjacent_concat = IS_CATEGORY(*cit, WhiteSpaceTokenType);
        }

        if (IS_CATEGORY((*cit), ParameterTokenType)) {
        // copy argument 'i' instead of the parameter token i
        typename ContainerT::size_type i;
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
        bool is_ellipsis = false;
        
            if (IS_EXTCATEGORY((*cit), ExtParameterTokenType)) {
                BOOST_SPIRIT_ASSERT(wave::need_variadics(ctx.get_language()));
                i = token_id(*cit) - T_EXTPARAMETERBASE;
                is_ellipsis = true;
            } 
            else 
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
            {
                i = token_id(*cit) - T_PARAMETERBASE;
            }
            
            BOOST_SPIRIT_ASSERT(i < arguments.size());
            if (use_replaced_arg) {
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                if (is_ellipsis) {
                typename token_t::position_t const &pos = (*cit).get_position();

                    BOOST_SPIRIT_ASSERT(wave::need_variadics(ctx.get_language()));
                    impl::replace_ellipsis(expanded_args, i, expanded, pos);
                }
                else 
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                {
                // replace argument
                ContainerT const &arg = expanded_args[i];
                
                    std::copy(arg.begin(), arg.end(), 
                        std::inserter(expanded, expanded.end()));
                }
            }
            else if (adjacent_stringize && 
                    !IS_CATEGORY(*cit, WhiteSpaceTokenType)) 
            {
            // stringize the current argument
                BOOST_SPIRIT_ASSERT(!arguments[i].empty());
                
            typename token_t::position_t const pos =
                (*arguments[i].begin()).get_position();

#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                if (is_ellipsis && wave::need_variadics(ctx.get_language())) {
                    trim_argument_left(arguments[i]);
                    trim_argument_right(arguments.back());
                    expanded.push_back(token_t(T_STRINGLIT, 
                        impl::as_stringlit(arguments, i, pos), pos));
                }
                else {
                    trim_argument(arguments[i]);
                    expanded.push_back(token_t(T_STRINGLIT, 
                        impl::as_stringlit(arguments[i], pos,
                            wave::need_variadics(ctx.get_language())), pos));
                }
#else
                trim_argument(arguments[i]);
                expanded.push_back(token_t(T_STRINGLIT, 
                    impl::as_stringlit(arguments[i], pos), pos));
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)

                adjacent_stringize = false;
            }
            else {
            // simply copy the original argument (adjacent '##' or '#')
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                if (is_ellipsis) {
                typename token_t::position_t const &pos = (*cit).get_position();

                    trim_argument_left(arguments[i]);
                    trim_argument_right(arguments.back());
                    BOOST_SPIRIT_ASSERT(wave::need_variadics(ctx.get_language()));
                    impl::replace_ellipsis(arguments, i, expanded, pos);
                }
                else
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                {
                ContainerT &arg = arguments[i];
                
                    trim_argument(arg);
                    std::copy(arg.begin(), arg.end(), 
                        std::inserter(expanded, expanded.end()));
                }
            }
        }
        else if (!adjacent_stringize || T_POUND != id) {
        // insert the actual replacement token (if it is not the '#' operator)
            expanded.push_back(*cit);
        }
    }

    if (adjacent_stringize) {
    // error, '#' should not be the last token
        CPP_THROW(preprocess_exception, ill_formed_operator,
            "stringize ('#')", main_pos);
    }
        
// handle the cpp.concat operator
    if (seen_concat)
        concat_tokensequence(expanded);
}

template <typename ContextT, typename TraceT>
template <typename ContainerT>
inline void 
macromap<ContextT, TraceT>::rescan_replacement_list(token_t const &curr_token, 
    macro_definition<token_t> &macro_def, ContainerT &replacement_list, 
    ContainerT &expanded, bool expand_undefined, bool *seen_qualified_name)
{
    using namespace wave::cpplexer;

    if (!replacement_list.empty()) {
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
    // remove the placemarkers
        if (wave::need_variadics(ctx.get_language())) {
        typename ContainerT::iterator end = replacement_list.end();
        typename ContainerT::iterator it = replacement_list.begin();
        
            while (it != end) {
                if (T_PLACEMARKER == token_id(*it)) {
                typename ContainerT::iterator placemarker = it;
                
                    ++it;
                    replacement_list.erase(placemarker);
                }
                else {
                    ++it;
                }
            }
        }
#endif

    // rescan the replacement list, during this rescan the current macro under
    // expansion isn't available as an expandable macro
    on_exit::reset<bool> on_exit(macro_def.is_available_for_replacement, false);

        expand_whole_tokensequence(expanded, replacement_list.begin(),
            replacement_list.end(), expand_undefined, seen_qualified_name);

    // trim replacement list, leave placeholder tokens untouched
        trim_replacement_list(expanded);
    }
    
    if (expanded.empty()) {
    // the resulting replacement list should contain at least a placeholder
    // token
        BOOST_SPIRIT_ASSERT(expanded.empty());
        expanded.push_back(token_t(T_PLACEHOLDER, "_", curr_token.get_position()));
    }
}

template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT>
inline bool 
macromap<ContextT, TraceT>::expand_macro(ContainerT &expanded, 
    token_t const &curr_token, IteratorT &first, IteratorT const &last, 
    bool expand_undefined, 
    symbol_table<typename ContextT::token_t::string_t, 
        macro_definition<typename ContextT::token_t> > *scope,
    ContainerT *queue_symbol, bool *seen_qualified_name) 
{
    using namespace wave::cpplexer;
    
    if (0 == scope) scope = current_macros;
    
    BOOST_SPIRIT_ASSERT(T_IDENTIFIER == token_id(curr_token) ||
        IS_CATEGORY(token_id(curr_token), KeywordTokenType));
    typename defined_macros_t::iterator it = 
        scope->find(curr_token.get_value());
        
    if (it == scope->end()) {
        ++first;    // advance

    // try to expand a predefined macro (__FILE__, __LINE__ or __INCLUDE_LEVEL__)
        if (expand_predefined_macro(curr_token, expanded))
            return false;
        
    // not defined as a macro
        if (0 != queue_symbol) {
            expanded.splice(expanded.end(), *queue_symbol);
        }
        else {
            expanded.push_back(curr_token);
        }
        return false;
    }

// ensure the parameters to be replaced with special parameter tokens
#if !defined(WAVE_USE_TST_SYMBOLTABLE)
macro_definition<token_t> &macro_def = *(*it).second.get();
#else
macro_definition<token_t> &macro_def = (*it).data();
#endif // !defined(WAVE_USE_TST_SYMBOLTABLE)

    macro_def.replace_parameters();

// test if this macro is currently available for replacement
    if (!macro_def.is_available_for_replacement) {
    // this macro is marked as non-replaceable
    // copy the macro name itself
        if (0 != queue_symbol) {
            queue_symbol->push_back(token_t(T_NONREPLACABLE_IDENTIFIER,
                curr_token.get_value(), curr_token.get_position()));
            expanded.splice(expanded.end(), *queue_symbol);
        }
        else {
            expanded.push_back(token_t(T_NONREPLACABLE_IDENTIFIER,
                curr_token.get_value(), curr_token.get_position()));
        }
        ++first;
        return false;
    }

// try to replace the current identifier as a function-like macro
ContainerT replacement_list;

    if (T_LEFTPAREN == impl::next_token<IteratorT>::peek(first, last)) {
    // called as a function-like macro 
        skip_to_token(first, last, T_LEFTPAREN);
        
        if (macro_def.is_functionlike) {
        // defined as a function-like macro
        
        // collect the arguments
        std::vector<ContainerT> arguments;
        typename std::vector<ContainerT>::size_type count_args = 
            collect_arguments (curr_token, arguments, first, last, 
                macro_def.macroparameters.size());

        // verify the parameter count
            if (count_args < macro_def.macroparameters.size() ||
                arguments.size() < macro_def.macroparameters.size()) 
            {
            // too few macro arguments
                CPP_THROW(preprocess_exception, too_few_macroarguments, 
                    curr_token.get_value(), main_pos);
            }
            
            if (count_args > macro_def.macroparameters.size() ||
                arguments.size() > macro_def.macroparameters.size()) 
            {
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                if (!macro_def.has_ellipsis) 
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
                {
                // too many macro arguments
                    CPP_THROW(preprocess_exception, too_many_macroarguments, 
                        curr_token.get_value(), main_pos);
                }
            }
                
        // inject tracing support
            trace.expanding_function_like_macro(macro_def.macroname, 
                macro_def.macroparameters, macro_def.macrodefinition,
                curr_token, arguments);
        
        // try to expand the arguments itself
        std::vector<ContainerT> expanded_args(arguments.size());
        
            expand_arguments(arguments, expanded_args, expand_undefined);
            
        // expand the replacement list of this macro
            expand_replacement_list(macro_def, arguments, expanded_args,
                replacement_list);
        }
        else {
        // defined as an object-like macro
            trace.expanding_object_like_macro(macro_def.macroname, 
                macro_def.macrodefinition, curr_token);

            std::copy (macro_def.macrodefinition.begin(), 
                macro_def.macrodefinition.end(), 
                std::inserter(replacement_list, replacement_list.end()));
        }
    }
    else {
    // called as an object like macro
#if !defined(WAVE_USE_TST_SYMBOLTABLE)
        if ((*it).second->is_functionlike) {
#else
        if ((*it).data().is_functionlike) {
#endif // !defined(WAVE_USE_TST_SYMBOLTABLE)
        // defined as a function-like macro
            if (0 != queue_symbol) {
                queue_symbol->push_back(curr_token);
                expanded.splice(expanded.end(), *queue_symbol);
            }
            else {
                expanded.push_back(curr_token);
            }
            ++first;                // skip macro name
            return false;           // no further preprocessing required
        }
        else {
        // defined as an object-like macro (expand it)
            trace.expanding_object_like_macro(macro_def.macroname, 
                macro_def.macrodefinition, curr_token);

            std::copy (macro_def.macrodefinition.begin(), 
                macro_def.macrodefinition.end(), 
                std::inserter(replacement_list, replacement_list.end()));

            ++first;                // skip macro name
        }
    }

// rescan the replacement list
ContainerT expanded_list;

    trace.expanded_macro(replacement_list);
    
    rescan_replacement_list(curr_token, macro_def, replacement_list, 
        expanded_list, expand_undefined, seen_qualified_name);
    
    trace.rescanned_macro(expanded_list);  
    expanded.splice(expanded.end(), expanded_list);
    return true;        // rescan is required
}

template <typename ContextT, typename TraceT>
template <typename ContainerT>
inline bool 
macromap<ContextT, TraceT>::expand_predefined_macro(token_t const &curr_token, 
    ContainerT &expanded)
{
    using namespace wave::cpplexer;
    
string_t const &value = curr_token.get_value();

    if (value.size() < 8 || '_' != value[0] || '_' != value[1])
        return false;       // quick check failed
        
    if (value == "__LINE__") { 
    // expand the __LINE__ macro
    char buffer[22];    // 21 bytes holds all NUL-terminated unsigned 64-bit numbers

        using namespace std;    // for some systems sprintf is in namespace std
        sprintf(buffer, "%d", main_pos.get_line());
        expanded.push_back(token_t(T_INTLIT, buffer, curr_token.get_position()));
        return true;
    }
    else if (value == "__FILE__") {
    // expand the __FILE__ macro
        namespace fs = boost::filesystem;
        
    std::string file("\"");
    fs::path filename(main_pos.get_file().c_str(), fs::native);
    
        using wave::util::impl::escape_lit;
        file += escape_lit(filename.native_file_string()) + "\"";
        expanded.push_back(token_t(T_STRINGLIT, file.c_str(), 
            curr_token.get_position()));
        return true;
    }
    else if (value == "__INCLUDE_LEVEL__") {
    // expand the __INCLUDE_LEVEL__ macro
    char buffer[22];    // 21 bytes holds all NUL-terminated unsigned 64-bit numbers

        using namespace std;    // for some systems sprintf is in namespace std
        sprintf(buffer, "%d", ctx.get_iteration_depth());
        expanded.push_back(token_t(T_INTLIT, buffer, curr_token.get_position()));
        return true;
    }
    return false;   // no predefined token
}

///////////////////////////////////////////////////////////////////////////////
//
//  resolve_defined(): resolve the operator defined() and replace it with the 
//                     correct T_INTLIT token
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT>
inline typename ContextT::token_t const &
macromap<ContextT, TraceT>::resolve_defined(IteratorT &first, 
    IteratorT const &last, ContainerT &pending) 
{
    using namespace wave::cpplexer;
    using namespace wave::grammars;

ContainerT result;
IteratorT start = first;
boost::spirit::parse_info<IteratorT> hit = 
    defined_grammar_gen<token_t>::parse_operator_defined(start, last, result);
    
    if (!hit.hit) {
        CPP_THROW(preprocess_exception, ill_formed_expression, 
            "defined()", main_pos);
    }
    impl::assign_iterator<IteratorT>::do_(first, hit.stop);

// insert a token, which reflects the outcome
    pending.push_back(token_t(T_INTLIT, 
        is_defined(result.begin(), result.end()) ? "1" : "0", 
        main_pos));

on_exit::pop_front<token_t> pop_front_token(pending);

    return act_token = pending.front();
}

///////////////////////////////////////////////////////////////////////////////
//
//  resolve_operator_pragma(): resolve the operator _Pragma() and dispatch to
//                             the associated action
//
//      This function returns true, if the pragma was correctly interpreted. 
//      The iterator 'first' is positioned behind the closing ')'.
//      This function returnes false, if the _Pragma was not known, the 
//      preprocessed token sequence is pushed back to the 'pending' sequence.
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
template <typename IteratorT, typename ContainerT>
inline bool
macromap<ContextT, TraceT>::resolve_operator_pragma(IteratorT &first, 
    IteratorT const &last, ContainerT &pending) 
{
// isolate the parameter of the operator _Pragma
    token_t pragma_token = *first;
    
    ++first;
    std::vector<ContainerT> arguments;
    typename std::vector<ContainerT>::size_type count_args = 
        collect_arguments (pragma_token, arguments, first, last, 1);

// verify the parameter count
    if (0 == pragma_token.get_position().get_file().size())
        pragma_token.set_position(act_token.get_position());
        
    if (1 > count_args || 1 > arguments.size()) {
    // too few macro arguments
        CPP_THROW(preprocess_exception, too_few_macroarguments, 
            pragma_token.get_value(), pragma_token.get_position());
    }
    if (1 < count_args || 1 < arguments.size()) {
    // too many macro arguments
        CPP_THROW(preprocess_exception, too_many_macroarguments, 
            pragma_token.get_value(), pragma_token.get_position());
    }

// preprocess the pragma token body
    ContainerT expanded;
    expand_whole_tokensequence(expanded, arguments[0].begin(), 
        arguments[0].end(), false);

// unescape the parameter of the operator _Pragma
    typedef typename token_t::string_t string_t;
    
    string_t pragma_cmd;
    typename ContainerT::const_iterator end_exp = expanded.end();
    for (typename ContainerT::const_iterator it_exp = expanded.begin();
         it_exp != end_exp; ++it_exp)
    {
        if (T_EOF == token_id(*it_exp))
            break;
        if (IS_CATEGORY(*it_exp, WhiteSpaceTokenType))
            continue;
            
        if (T_STRINGLIT != token_id(*it_exp)) {
        // ill formed operator _Pragma
            CPP_THROW(preprocess_exception, ill_formed_pragma_option, "_Pragma", 
                pragma_token.get_position());
        }

        if (pragma_cmd.size() > 0) {
        // there should be exactly one string literal (string literals are to 
        // be concatenated at translation phase 6, but _Pragma operators are 
        // to be executed at translation phase 4)
            CPP_THROW(preprocess_exception, ill_formed_pragma_option, "_Pragma", 
                pragma_token.get_position());
        }
        
    // remove the '\"' and concat all given stringlit-values
        string_t token_str = (*it_exp).get_value();
        pragma_cmd += token_str.substr(1, token_str.size() - 2);
    }
    pragma_cmd = impl::unescape_lit(pragma_cmd);

// tokenize the pragma body
    typedef typename ContextT::lex_t lex_t;
    
    ContainerT pragma;
    std::string pragma_cmd_str(pragma_cmd.c_str());
    lex_t it = lex_t(pragma_cmd_str.begin(), pragma_cmd_str.end(), 
        pragma_token.get_position(), ctx.get_language());
    lex_t end = lex_t();
    for (/**/; it != end; ++it) 
        pragma.push_back(*it);

// analyze the preprocessed tokensequence and eventually dispatch to the 
// associated action
    if (interpret_pragma(ctx, pragma_token, pragma.begin(), pragma.end(), 
        pending, ctx.get_language()))
    {
        return true;    // successfully recognized a wave specific pragma
    }
    
// unknown pragma token sequence, push it back and return to the caller
    pending.push_front(token_t(T_SPACE, " ", pragma_token.get_position()));
    pending.push_front(token_t(T_RIGHTPAREN, ")", pragma_token.get_position()));
    pending.push_front(token_t(T_STRINGLIT, string_t("\"") + pragma_cmd + "\"",
        pragma_token.get_position()));
    pending.push_front(token_t(T_LEFTPAREN, "(", pragma_token.get_position()));
    pending.push_front(pragma_token);
    return false;
}

// Test if the result of a concat operator is well formed. 
template <typename ContextT, typename TraceT>
template <typename ContainerT>
inline bool
macromap<ContextT, TraceT>::is_invalid_concat(string_t new_value, 
    typename token_t::position_t const &pos, ContainerT &rescanned)
{
// retokenize the newly generated string
    typedef typename ContextT::lex_t lex_t;
    
    std::string value_to_test(new_value.c_str());
    lex_t it = lex_t(value_to_test.begin(), value_to_test.end(), pos, 
        ctx.get_language());
    lex_t end = lex_t();
    for (/**/; it != end; ++it) 
        rescanned.push_back(*it);
        
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    if (wave::need_cpp0x(ctx.get_language()))
        return false;       // in C++0x mode token pasting is well defined
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
        
// test if the newly generated token sequence contains more than 1 token
// the second one is the T_EOF token
    BOOST_SPIRIT_ASSERT(T_EOF == token_id(rescanned.back()));
    return rescanned.size() != 2;
}

template <typename ContextT, typename TraceT>
template <typename ContainerT>
inline void 
macromap<ContextT, TraceT>::concat_tokensequence(ContainerT &expanded)
{
    using namespace wave::cpplexer;
    typedef typename ContainerT::iterator iterator_t;
    
    iterator_t end = expanded.end();
    iterator_t prev = end;
    for (iterator_t it = expanded.begin(); it != end; /**/) 
    {
        if (T_POUND_POUND == token_id(*it)) {
        iterator_t next = it;
    
            ++next;
            if (prev == end || next == end) {
            // error, '##' should be in between two tokens
                CPP_THROW(preprocess_exception, ill_formed_operator,
                    "concat ('##')", main_pos);
            }

        // replace prev##next with the concatenated value, skip whitespace
        // before and after the '##' operator
            while (IS_CATEGORY(*next, WhiteSpaceTokenType)) {
                ++next;
                if (next == end) {
                // error, '##' should be in between two tokens
                    CPP_THROW(preprocess_exception, ill_formed_operator,
                        "concat ('##')", main_pos);
                }
            }
            
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
            if (wave::need_variadics(ctx.get_language())) {
                if (T_PLACEMARKER == token_id(*next)) {
                // remove the '##' and the next tokens from the sequence
                iterator_t first_to_delete = prev;

                    expanded.erase(++first_to_delete, ++next);
                    it = next;
                    continue;
                }
                else if (T_PLACEMARKER == token_id(*prev)) {
                // remove the '##' and the next tokens from the sequence
                iterator_t first_to_delete = prev;

                    *prev = *next;
                    expanded.erase(++first_to_delete, ++next); 
                    it = next;
                    continue;
                }
            }
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)

        // test if the concat operator has to concatenate two unrelated 
        // tokens i.e. the result yields more then one token
        string_t concat_result;
        ContainerT rescanned;

            concat_result = ((*prev).get_value() + (*next).get_value());

        // Here we have to work around a conflict between the Standards 
        // requirement, that the preprocessor has to act upon so called 
        // pp-tokens and the fact, that Wave acts upon C++ tokens. So we have 
        // eventually to combine the current token with the next tokens
        // if these are of type T_IDENTIFIER or T_INTLIT following without any 
        // interventing whitespace.
        iterator_t save = next;
        
            if (IS_CATEGORY(*prev, IdentifierTokenType)) {
            token_id id = impl::next_token<iterator_t>::peek(next, end, false);
            
                while (IS_CATEGORY(id, IdentifierTokenType) || 
                       IS_CATEGORY(id, KeywordTokenType) || T_INTLIT == id)
                {
                    concat_result += (*++next).get_value();
                    id = impl::next_token<iterator_t>::peek(next, end, false);
                }
            }

        // analyze the validity of the concatenation result
            if (is_invalid_concat(concat_result, (*prev).get_position(), 
                    rescanned) &&
                !IS_CATEGORY(*prev, WhiteSpaceTokenType) && 
                !IS_CATEGORY(*next, WhiteSpaceTokenType)) 
            {
            string_t error_string("\"");
            
                error_string += (*prev).get_value();
                error_string += "\" and \"";
                error_string += (*save).get_value();
                error_string += "\"";
                CPP_THROW(preprocess_exception, invalid_concat,
                    error_string, main_pos);
            }

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
            if (wave::need_cpp0x(ctx.get_language())) {
            // remove the prev, '##' and the next tokens from the sequence
                expanded.erase(prev, ++next);       // remove not needed tokens   

            // replace the old token (pointed to by *prev) with the retokenized
            // sequence
            typename ContainerT::reverse_iterator rit = rescanned.rbegin();

                BOOST_SPIRIT_ASSERT(rit != rescanned.rend());
                rescanned.erase((++rit).base());
                expanded.splice(next, rescanned);

            // the last token of the inserted sequence is the new previous
                prev = next;
                --prev;
            }
            else
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
            {
            // we leave the token_id unchanged, but unmark the token as 
            // disabled, if appropriate
                (*prev).set_value(concat_result);
                if (T_NONREPLACABLE_IDENTIFIER == token_id(*prev))
                    (*prev).set_token_id(T_IDENTIFIER);
                
            // remove the '##' and the next tokens from the sequence
            iterator_t first_to_delete = prev;

                expanded.erase(++first_to_delete, ++next); 
            }
            it = next;
            continue;
        }

    // save last non-whitespace token position
        if (!IS_CATEGORY(*it, WhiteSpaceTokenType))        
            prev = it;

        ++it;           // next token, please
    }
}

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
///////////////////////////////////////////////////////////////////////////////
//
//  experimental: macro scoping support
//
///////////////////////////////////////////////////////////////////////////////

///////////////////////////////////////////////////////////////////////////////
//
//  #region directive 
//
//    - syntax
//
//      #region [<identifier>]
//
//      Where <identifier> is a simple name (non-qualified name) defining the 
//      name of the region to open. This name is optional. If the name is 
//      ommitted a nameless region is opened. Macros defined inside a nameless
//      region may not be accessed from outside this region.
//
//    - The #region directive is opaque for all macro definitions made outside
//      this region. A #region may be re-opened (i.e. a #region directive with
//      the same name is found at the same scope), and macros from the first
//      occurence of this region will be visible.
//
//    - Region names and macro names of the same scope are stored into the same 
//      symbol table. I.e. at one scope there shall not be defined a region and 
//      a macro with the same name.
//
//    - #region's may be nested.
//
//    - The arguments of the #region directive are _not_ subject to macro 
//      expansion.
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
inline void 
macromap<ContextT, TraceT>::open_scope(string_t const &name,
    defined_macros_t *current_scope, 
    boost::shared_ptr<defined_macros_t> &new_scope)
{
    bool is_new_scope = false;
    boost::shared_ptr<defined_macros_t> next_scope;
    if (!current_scope -> get_scope(name, next_scope))
        is_new_scope = true;

    new_scope = current_scope -> begin_scope(name);
    if (is_new_scope) {
    // make predefined macros available
        init_predefined_macros(new_scope.get(), false);
        
    // predefine the __CURRENT_SCOPE__ and __FULL_SCOPE__ names
        typename token_t::position_t pos;
        predefine_macro(new_scope.get(), 
            "__STDC_CURRENT_" WAVE_PP_REGION_UC "__", 
            token_t(T_STRINGLIT, name, pos));
        predefine_macro(new_scope.get(),
            "__STDC_FULL_" WAVE_PP_REGION_UC "__", 
            token_t(T_STRINGLIT, new_scope->get_full_name(), pos));
    }
}

template <typename ContextT, typename TraceT>
inline void 
macromap<ContextT, TraceT>::begin_unnamed_scope()
{
// create/open a new unnamed sub-region
boost::shared_ptr<defined_macros_t> new_scope;

    open_scope("", current_macros, new_scope);
    scopes.push(new_scope);
    current_macros = new_scope.get();
}

template <typename ContextT, typename TraceT>
template <typename IteratorT>
inline void 
macromap<ContextT, TraceT>::begin_scope(IteratorT const &begin, 
    IteratorT const &end)
{
    using namespace cpplexer;
    
defined_macros_t *current_scope = 0;
bool was_colon_colon = false;
bool found_qualified_name = false;
    
    for (IteratorT it = begin; it != end; /**/) {
        if (token_id(*it) == T_COLON_COLON) {
            if (was_colon_colon) {
            // error: two consecutive '::'
                CPP_THROW(preprocess_exception, invalid_macroname, 
                    get_full_name(begin, end), main_pos);
            }
            if (0 == current_scope) {
            // initialize starting scope (global scope)
                if (current_macros -> get_contains_unnamed_part()) {
                    CPP_THROW(preprocess_exception, illegal_global_region,
                        get_full_name(begin, end), main_pos);
                }
                if (++it == end) {
                    scopes.push(defined_macros);    // store opened region
                    current_macros = defined_macros.get();
                    found_qualified_name = true;
                    break;
                }
                else {
                    current_scope = defined_macros.get();
                }
            } 
            else {
                ++it;
            }
            was_colon_colon = true;
        }
        else {
            if (0 == current_scope) {
            // initialize starting scope
                current_scope = current_macros;     // start at current scope
            }
            else if (!was_colon_colon) {
            // error: missing '::' between two parts of the macro name
                CPP_THROW(preprocess_exception, invalid_macroname, 
                    get_full_name(begin, end), main_pos);
            }
            was_colon_colon = false;
            
        // open the given sub-region (create it if nessecary)
        string_t name ((*it).get_value());
        boost::shared_ptr<defined_macros_t> next_scope;

            open_scope(name, current_scope, next_scope);
            if (++it == end) {
                scopes.push(next_scope);            // store opened region
                current_macros = next_scope.get();
                found_qualified_name = true;
                break;
            }
            else {
            // analyze the next part of the qualified name
                current_scope = next_scope.get();
            }
        }
    }
    
    if (!found_qualified_name) {
    // missing last part of the name (scope end with '::')
        CPP_THROW(preprocess_exception, invalid_macroname, 
            get_full_name(begin, end), main_pos);
    }
}

///////////////////////////////////////////////////////////////////////////////
//
//  #endregion
//
//    - syntax
//
//      #endregion
//
//    - The #endregion ends the last opened macro region. It is directive 
//      opaque for all macros defined inside the closed region. Macros from 
//      this region may be accessed outside this region only if imported 
//      (see #import) or if used as qualified names specifying the region and 
//      the macro name.
//
///////////////////////////////////////////////////////////////////////////////
template <typename ContextT, typename TraceT>
inline void 
macromap<ContextT, TraceT>::end_scope()
{
    if (0 == scopes.size()) {
    // should not be used at global scope
        CPP_THROW(preprocess_exception, unexpected_endregion, "", main_pos);
    }
    
// inform the symbol table
    current_macros->end_scope();

// restore the previous scope
    scopes.pop();
    if (scopes.size() > 0)
        current_macros = scopes.top().get();
    else
        current_macros = defined_macros.get();
}

///////////////////////////////////////////////////////////////////////////////
//
//  #import directive
//
//    - syntax
//
//      #import <qualified-name> [, <qualified-name>]
//
//      Where <qualified-name> is a macro name or region name consisting out 
//      of a possibly empty sequence of region names interspersed with '::' 
//      optionally tokens followed by a macro name defined inside the 
//      specified region.
//      
//      If <qualified-name> refers to a macro, then the referenced macro 
//      definition is _copied_ into the current region, just if it were defined 
//      here initially.
//
//      If <qualified-name> refers to a region, then
//      - all macro definitions of the referenced region are _copied_ into the
//        current region, just if these were defined here initially.
//      - all sub-regions of the referenced region are made available from the
//        current region, just if these were defined as direct sub-regions of 
//        the current region.
//
//      Imported macros may be undefined with #undef as usual. This removes
//      the referenced macro from the current region, but leaves it unchanged 
//      in the original region.
//
//    - The arguments of the #import directive are _not_ subject to macro 
//      expansion.
//
///////////////////////////////////////////////////////////////////////////////
namespace {

    ///////////////////////////////////////////////////////////////////////////
    //
    //  get the full name of a given macro name or scope name (concatenate the
    //  string representations of the single tokens).
    //
    template <typename IteratorT>
    inline std::string
    get_full_name(IteratorT const &begin, IteratorT const &end)
    {
    std::string full_name;
    
        for (IteratorT err_it = begin; err_it != end; ++err_it) 
            full_name += (*err_it).get_value().c_str();

        return full_name;
    }
    
    template <typename IteratorT>
    inline bool
    is_qualified_name(IteratorT const &begin, IteratorT const &end)
    {
    bool was_colon_colon = false;

        for (IteratorT it = begin; it != end; ++it) {
            if (token_id(*it) == T_COLON_COLON) {
                if (was_colon_colon) {
                // error: two consecutive '::'
                    return false;
                }
                was_colon_colon = true;
            }
            else if (T_IDENTIFIER == token_id(*it) || 
                IS_CATEGORY(*it, KeywordTokenType)) 
            {
                if (!was_colon_colon) {
                // error: missing '::' between two parts of the macro name
                    return false;
                }
                was_colon_colon = false;
            }
            else if (!IS_CATEGORY(*it, WhiteSpaceTokenType)) {
            // invalid token type inside a qualified name
                return false;
            }
        }
        return true;
    }
}

template <typename ContextT, typename TraceT>
template <typename IteratorT>
inline void 
macromap<ContextT, TraceT>::import_name(IteratorT const &begin, 
    IteratorT const &end)
{
    using namespace cpplexer;
    defined_macros_t *current_scope = 0;
    bool was_colon_colon = false;
    
    for (IteratorT it = begin; it != end; /**/) {
        if (token_id(*it) == T_COLON_COLON) {
            if (was_colon_colon) {
            // error: two consecutive '::'
                CPP_THROW(preprocess_exception, invalid_macroname, 
                    get_full_name(begin, end), main_pos);
            }
            if (0 == current_scope) {
            // initialize starting scope (global scope)
                if (++it == end) {
                // '#import ::' specified
                    current_macros -> import_scope(defined_macros.get(), 
                        main_pos); 
                    return;     // imported scope
                }
                current_scope = defined_macros.get();
            }
            else {
                ++it;
            }
            was_colon_colon = true;
        }
        else {
            if (0 == current_scope) {
            // initialize starting scope
                current_scope = current_macros;     // start at current scope
            }
            else if (!was_colon_colon) {
            // error: missing '::' between two parts of the macro name
                CPP_THROW(preprocess_exception, invalid_macroname, 
                    get_full_name(begin, end), main_pos);
            }
            was_colon_colon = false;
            
        string_t name ((*it).get_value());
        boost::shared_ptr<defined_macros_t> next_scope;
        typename defined_macros_t::const_iterator cit = 
            current_scope -> find(name);
        
            if (cit != current_scope -> end()) {
            // defined as a macro name
            
            // should be the last given name part 
                if (++it != end) {
                    CPP_THROW(preprocess_exception, undefined_macroname, 
                        get_full_name(begin, end), main_pos);
                }

            // import this macro name
#if !defined(WAVE_USE_TST_SYMBOLTABLE)
                if (!current_macros -> import_macroname(name, (*cit).second)) {
#else
                if (!current_macros -> import_macroname(name, (*cit).data())) {
#endif // !defined(WAVE_USE_TST_SYMBOLTABLE)
                // this macro name is already used as a scope name here
                    CPP_THROW(preprocess_exception, alreadydefined_name, 
                        name, main_pos);
                }
                return;     // imported macro
            }
            else if (current_scope -> get_scope(name, next_scope)) {
            // if this part was the last one, then import the whole scope
                if (++it == end) {
                    current_macros -> import_scope(next_scope.get(), main_pos); 
                    return;     // imported scope
                }
                else {
                // defined as a scope name
                    current_scope = next_scope.get();
                }                
            }
            else {
            // name is not defined
                CPP_THROW(preprocess_exception, undefined_macroname, 
                    get_full_name(begin, end), main_pos);
            }
        }
    }
}
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)

///////////////////////////////////////////////////////////////////////////////
//
//  init_predefined_macros(): init the predefined macros
//
///////////////////////////////////////////////////////////////////////////////

namespace predefined_macros {

// list of static predefined macros
    struct static_macros {
        char const *name;
        wave::cpplexer::token_id token_id;
        char const *value;
    };

// C++ mode 
    static_macros static_data_cpp[] = {
        { "__STDC__", T_INTLIT, "1" },
        { "__cplusplus", T_INTLIT, "199711L" },
        { 0, T_EOF, 0 }
    }; 

#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
// C99 mode
    static_macros static_data_c99[] = {
        { "__STDC__", T_INTLIT, "1" },
        { "__STDC_VERSION__", T_INTLIT, "199901L" },
        { "__STDC_HOSTED__", T_INTLIT, "0" },
        { "__WAVE_HAS_VARIADICS__", T_INTLIT, "1" },
        { 0, T_EOF, 0 }
    }; 

#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
// C++0x mode 
    static_macros static_data_cpp0x[] = {
        { "__STDC__", T_INTLIT, "1" },
        { "__cplusplus", T_INTLIT, "200304L" },
        { "__WAVE_HAS_VARIADICS__", T_INTLIT, "1" },
        { 0, T_EOF, 0 }
    }; 
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
    
// list of dynamic predefined macros
    typedef char const * (* get_dynamic_value)(bool);

// __DATE__ 
    inline 
    char const *get_date(bool reset)
    {
    static const char *const monthnames[] =
    {
        "Jan", "Feb", "Mar", "Apr", "May", "Jun",
        "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
    };
    static std::string datestr;
    
        if (reset) {
            datestr.erase();
        }
        else {
        // for some systems sprintf, time_t etc. is in namespace std
            using namespace std;    

        time_t tt = time(0);
        struct tm *tb = 0;

            if (tt != (time_t)-1) {
            char buffer[sizeof("\"Oct 11 1347\"")+1];

                tb = localtime (&tt);
                sprintf (buffer, "\"%s %2d %4d\"",
                    monthnames[tb->tm_mon], tb->tm_mday, tb->tm_year + 1900);
                datestr = buffer;
            }
            else {
                datestr = "\"??? ?? ????\"";
            }
        }
        return datestr.c_str();
    }

// __TIME__
    inline 
    char const *get_time(bool reset)
    {
    static std::string timestr;
    
        if (reset) {
            timestr.erase();
        }
        else {
        // for some systems sprintf, time_t etc. is in namespace std
            using namespace std;    

        time_t tt = time(0);
        struct tm *tb = 0;

            if (tt != (time_t)-1) {
            char buffer[sizeof("\"12:34:56\"")+1];

                tb = localtime (&tt);
                sprintf (buffer, "\"%02d:%02d:%02d\"", tb->tm_hour, 
                    tb->tm_min, tb->tm_sec);
                timestr = buffer;
            }
            else {
                timestr = "\"??:??:??\"";
            }
        }
        return timestr.c_str();
    }

// __SPIRIT_PP__/__WAVE__
    inline 
    char const *get_version(bool /*reset*/)
    {
    static std::string versionstr;
    char buffer[sizeof("0x0000")+1];

        using namespace std;    // for some systems sprintf is in namespace std
        sprintf(buffer, "0x%02d%1d%1d", WAVE_VERSION_MAJOR, 
            WAVE_VERSION_MINOR, WAVE_VERSION_SUBMINOR);
        versionstr = buffer;
        return versionstr.c_str();
    }
    
// __SPIRIT_PP_VERSION__/__WAVE_VERSION__
    wave::util::time_conversion_helper const 
        compilation_time(__DATE__ " " __TIME__);
        
    inline 
    char const *get_fullversion(bool /*reset*/)
    {
    static std::string versionstr;
    char buffer[sizeof("0x00000000")+1];

    // for some systems sprintf, time_t etc. is in namespace std
        using namespace std;    

    // calculate the number of days since Dec 13 2001 
    // (the day the cpp project was started)
    tm first_day;

        using namespace std;    // for some systems memset is in namespace std
        memset (&first_day, 0, sizeof(tm));
        first_day.tm_mon = 11;           // Dec
        first_day.tm_mday = 13;          // 13
        first_day.tm_year = 101;         // 2001

    long seconds = long(difftime(compilation_time.get_time(), 
        mktime(&first_day)));

        sprintf(buffer, "0x%02d%1d%1d%04ld", WAVE_VERSION_MAJOR,
             WAVE_VERSION_MINOR, WAVE_VERSION_SUBMINOR, seconds/(3600*24));
        versionstr = buffer;
        return versionstr.c_str();
    }
    
// __SPIRIT_PP_VERSION_STR__/__WAVE_VERSION_STR__
    inline 
    char const *get_versionstr(bool /*reset*/)
    {
    static std::string versionstr;
    char buffer[sizeof("\"00.00.00.0000\"")+1];

    // for some systems sprintf, time_t etc. is in namespace std
        using namespace std;    

    // calculate the number of days since Dec 13 2001 
    // (the day the cpp project was started)
    tm first_day;

        using namespace std;    // for some systems memset is in namespace std
        memset (&first_day, 0, sizeof(tm));
        first_day.tm_mon = 11;           // Dec
        first_day.tm_mday = 13;          // 13
        first_day.tm_year = 101;         // 2001

    long seconds = long(difftime(compilation_time.get_time(), 
        mktime(&first_day)));

        sprintf(buffer, "\"%d.%d.%d.%ld\"", WAVE_VERSION_MAJOR,
             WAVE_VERSION_MINOR, WAVE_VERSION_SUBMINOR, seconds/(3600*24));
        versionstr = buffer;
        return versionstr.c_str();
    }
    
    struct dynamic_macros {
        char const *name;
        wave::cpplexer::token_id token_id;
        get_dynamic_value generator;
    }
    dynamic_data[] = {
        { "__DATE__", T_STRINGLIT, get_date },
        { "__TIME__", T_STRINGLIT, get_time },
        { "__SPIRIT_PP__", T_INTLIT, get_version },
        { "__SPIRIT_PP_VERSION__", T_INTLIT, get_fullversion },
        { "__SPIRIT_PP_VERSION_STR__", T_STRINGLIT, get_versionstr },
        { "__WAVE__", T_INTLIT, get_version },
        { "__WAVE_VERSION__", T_INTLIT, get_fullversion },
        { "__WAVE_VERSION_STR__", T_STRINGLIT, get_versionstr },
        { 0, T_EOF, 0 }
    };

}   // namespace predefined_macros

template <typename ContextT, typename TraceT>
inline void
macromap<ContextT, TraceT>::predefine_macro(defined_macros_t *scope, 
    string_t const &name, token_t const &t)
{
std::list<token_t> macrodefinition;
std::vector<token_t> param;

    macrodefinition.push_back(t);
    add_macro(token_t(T_IDENTIFIER, name, t.get_position()), 
        false, param, macrodefinition, true, scope);
}

template <typename ContextT, typename TraceT>
inline void 
macromap<ContextT, TraceT>::init_predefined_macros(defined_macros_t *scope,
    bool at_global_scope)
{
    using namespace predefined_macros;

// if no scope is given, use the current one
defined_macros_t *current_scope = scope ? scope : current_macros;

// first, add the static macros
typename token_t::position_t pos;

#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
    if (wave::need_c99(ctx.get_language())) {
    // define C99 specifics
        for (int i = 0; 0 != static_data_c99[i].name; ++i) {
            predefine_macro(current_scope, static_data_c99[i].name,
                token_t(static_data_c99[i].token_id, 
                    static_data_c99[i].value, pos));
        }
    }
    else 
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
    if (wave::need_cpp0x(ctx.get_language())) {
    // define C++0x specifics
        for (int i = 0; 0 != static_data_cpp0x[i].name; ++i) {
            predefine_macro(current_scope, static_data_cpp0x[i].name, 
                token_t(static_data_cpp0x[i].token_id, 
                    static_data_cpp0x[i].value, pos));
        }

    // add the global scope definitions
        if (at_global_scope) {
        typename token_t::position_t pos;

            predefine_macro(current_scope, "__STDC_GLOBAL__", 
                token_t(T_INTLIT, "1", pos));
            predefine_macro(current_scope, 
                "__STDC_CURRENT_" WAVE_PP_REGION_UC "__",  
                token_t(T_STRINGLIT, "", pos));
            predefine_macro(current_scope, 
                "__STDC_FULL_" WAVE_PP_REGION_UC "__", 
                token_t(T_STRINGLIT, "::", pos));
        }
    }
    else
#endif // defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
    {
    // define C++ specifics
        for (int i = 0; 0 != static_data_cpp[i].name; ++i) {
            predefine_macro(current_scope, static_data_cpp[i].name, 
                token_t(static_data_cpp[i].token_id, 
                    static_data_cpp[i].value, pos));
        }
        
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
    // define __WAVE_HAS_VARIADICS__, if appropriate
        if (wave::need_variadics(ctx.get_language())) {
            predefine_macro(current_scope, "__WAVE_HAS_VARIADICS__",
                token_t(T_INTLIT, "1", pos));
        }
#endif // defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
    }
    
// now add the dynamic macros
    for (int j = 0; 0 != dynamic_data[j].name; ++j) {
        predefine_macro(current_scope, dynamic_data[j].name,
            token_t(dynamic_data[j].token_id, 
                dynamic_data[j].generator(false), pos));
    }
}

template <typename ContextT, typename TraceT>
inline void 
macromap<ContextT, TraceT>::reset_macromap()
{
    current_macros->clear();
    predefined_macros::get_time(true);
    predefined_macros::get_date(true);
    act_token = token_t();
}

///////////////////////////////////////////////////////////////////////////////
}   // namespace util
}   // namespace wave

#endif // !defined(CPP_MACROMAP_HPP_CB8F51B0_A3F0_411C_AEF4_6FF631B8B414_INCLUDED)

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here


Written By
United States United States
Actively involved in Boost and the development of the Spirit parser construction framework.

Comments and Discussions