Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Wave: a Standard conformant C++ preprocessor library

, 10 Jan 2004
Describes a free and fully Standard conformant C++ preprocessor library
wave_preprocessor_demo.zip
wave_preprocessor_demo1.zip
wave_preprocessor_src.zip
wave
doc
theme
bkd.gif
bkd2.gif
bullet.gif
l_arr.gif
l_arr_disabled.gif
r_arr.gif
r_arr_disabled.gif
uc.gif
u_arr.gif
wave.gif
test
boost-build.jam
cpp_tokens
Jamfile.v2
list_includes
project-root.jam
wave
Jamfile
Jamfile.v2
runtests.sh
test_files
wave
cpplexer
re2clex
cpp.re
test
lextest.in
lextest.output
run_tests.sh
test_lexer.in
test_lexer.output
slex
test
lextest.in
lextest.re2c.output
lextest.slex.output
run_tests.sh
grammars
idllexer
re2clex
idl.re
util
wave_preprocessor_src1.zip
bkd.gif
bkd2.gif
bullet.gif
l_arr.gif
l_arr_disabled.gif
r_arr.gif
r_arr_disabled.gif
u_arr.gif
uc.gif
wave.gif
boost-build.jam
Jamfile.v2
project-root.jam
Jamfile
Jamfile.v2
runtests.sh
cpp.re
lextest.in
lextest.output
run_tests.sh
test_lexer.in
test_lexer.output
lextest.in
lextest.re2c.output
lextest.slex.output
run_tests.sh
/*=============================================================================
    Wave: A Standard compliant C++ preprocessor

    Detect the need to insert a whitespace token into the output stream
    
    Copyright (c) 2001-2004 Hartmut Kaiser
    http://spirit.sourceforge.net/

    Use, modification and distribution is subject to the Boost Software
    License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
    http://www.boost.org/LICENSE_1_0.txt)

    See Copyright.txt for full acknowledgements.
=============================================================================*/
#if !defined(INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED)
#define INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED

#include "wave/token_ids.hpp"   

///////////////////////////////////////////////////////////////////////////////
namespace wave {
namespace util {

namespace impl {

// T_IDENTIFIER
    template <typename StringT>
    inline bool
    would_form_universal_char (StringT const &value)
    {
        if ('u' != value[0] && 'U' != value[0])
            return false;
        if ('u' == value[0] && value.size() < 5)
            return false;
        if ('U' == value[0] && value.size() < 9)
            return false;
    
    typename StringT::size_type pos = 
        value.find_first_not_of("0123456789abcdefABCDEF", 1);
        
        if (StringT::npos == pos || 
            ('u' == value[0] && pos > 5) ||
            ('U' == value[0] && pos > 9))
        {
            return true;        // would form an universal char
        }
        return false;
    }
    template <typename StringT>
    inline bool 
    handle_identifier(WAVE_LEXER_NS::token_id prev, 
        WAVE_LEXER_NS::token_id before, StringT const &value)
    {
        using namespace WAVE_LEXER_NS;
        switch (static_cast<unsigned int>(prev)) {
        case T_IDENTIFIER:
        case T_NONREPLACABLE_IDENTIFIER:
        case T_INTLIT:
        case T_FLOATLIT:
#if !defined(WAVE_USE_RE2C_IDL_LEXER)
        case T_COMPL_ALT:
        case T_OR_ALT:
        case T_AND_ALT:
        case T_NOT_ALT:
        case T_XOR_ALT:
        case T_ANDASSIGN_ALT:
        case T_ORASSIGN_ALT:
        case T_XORASSIGN_ALT:
        case T_NOTEQUAL_ALT:
#else
        case T_FIXEDPOINTLIT:
#endif // !defined(WAVE_USE_RE2C_IDL_LEXER)
            return true;

         // avoid constructing universal characters (\u1234)
        case TOKEN_FROM_ID('\\', UnknownTokenType):
            return would_form_universal_char(value);
        }
        return false;
    }
// T_INTLIT
    inline bool 
    handle_intlit(WAVE_LEXER_NS::token_id prev, WAVE_LEXER_NS::token_id before)
    {
        using namespace WAVE_LEXER_NS;
        switch(prev) {
        case T_IDENTIFIER:
        case T_NONREPLACABLE_IDENTIFIER:
        case T_INTLIT:
        case T_FLOATLIT:
#if defined(WAVE_USE_RE2C_IDL_LEXER)
        case T_FIXEDPOINTLIT:
#endif // defined(WAVE_USE_RE2C_IDL_LEXER)
            return true;
        }
        return false;
    }
// T_FLOATLIT
    inline bool 
    handle_floatlit(WAVE_LEXER_NS::token_id prev, 
        WAVE_LEXER_NS::token_id before)
    {
        using namespace WAVE_LEXER_NS;
        switch(prev) {
        case T_IDENTIFIER:
        case T_NONREPLACABLE_IDENTIFIER:
        case T_INTLIT:
        case T_FLOATLIT:
#if defined(WAVE_USE_RE2C_IDL_LEXER)
        case T_FIXEDPOINTLIT:
#endif // defined(WAVE_USE_RE2C_IDL_LEXER)
            return true;
        }
        return false;
    }
#if !defined(WAVE_USE_RE2C_IDL_LEXER)
// <% T_LEFTBRACE
    inline bool 
    handle_alt_leftbrace(WAVE_LEXER_NS::token_id prev, 
        WAVE_LEXER_NS::token_id before)
    {
        using namespace WAVE_LEXER_NS;
        switch(prev) {
        case T_LESS:        // <<%
        case T_SHIFTLEFT:   // <<<%
            return true;
        }
        return false;
    }
// <: T_LEFTBRACKET
    inline bool 
    handle_alt_leftbracket(WAVE_LEXER_NS::token_id prev, 
        WAVE_LEXER_NS::token_id before)
    {
        using namespace WAVE_LEXER_NS;
        switch(prev) {
        case T_LESS:        // <<:
        case T_SHIFTLEFT:   // <<<:
            return true;
        }
        return false;
    }
#else
// T_FIXEDPOINTLIT
    inline bool 
    handle_fixedpointlit(WAVE_LEXER_NS::token_id prev, 
        WAVE_LEXER_NS::token_id before)
    {
        using namespace WAVE_LEXER_NS;
        switch(prev) {
        case T_IDENTIFIER:
        case T_NONREPLACABLE_IDENTIFIER:
        case T_INTLIT:
        case T_FLOATLIT:
        case T_FIXEDPOINTLIT:
            return true;
        }
        return false;
    }
#endif // !defined(WAVE_USE_RE2C_IDL_LEXER)
// T_DOT
    inline bool 
    handle_dot(WAVE_LEXER_NS::token_id prev, WAVE_LEXER_NS::token_id before)
    {
        using namespace WAVE_LEXER_NS;
        switch(prev) {
        case T_DOT:
            if (T_DOT == before)
                return true;    // ...
            break;
        }
        return false;
    }
// T_QUESTION_MARK
    inline bool 
    handle_questionmark(WAVE_LEXER_NS::token_id prev, 
        WAVE_LEXER_NS::token_id before)
    {
        using namespace WAVE_LEXER_NS;
        switch(static_cast<unsigned int>(prev)) {
        case TOKEN_FROM_ID('\\', UnknownTokenType):     // \?
        case T_QUESTION_MARK:   // ??
            return true;
        }
        return false;
    }
// T_NEWLINE
    inline bool
    handle_newline(WAVE_LEXER_NS::token_id prev, 
        WAVE_LEXER_NS::token_id before)
    {
        using namespace WAVE_LEXER_NS;
        switch(static_cast<unsigned int>(prev)) {
        case TOKEN_FROM_ID('\\', UnknownTokenType): // \ \n
        case T_DIVIDE:
            if (T_QUESTION_MARK == before)
                return true;    // ?/\n     // may be \\n
            break;
        }
        return false;
    }
    
}   // namespace impl

class insert_whitespace_detection 
{
public:
    insert_whitespace_detection() 
    :   prev(WAVE_LEXER_NS::T_EOF), beforeprev(WAVE_LEXER_NS::T_EOF) 
    {}
    
    template <typename StringT>
    bool must_insert(WAVE_LEXER_NS::token_id current, StringT const &value)
    {
        using namespace WAVE_LEXER_NS;
        switch (current) {
        case T_NONREPLACABLE_IDENTIFIER:
        case T_IDENTIFIER: 
            return impl::handle_identifier(prev, beforeprev, value); 
        case T_INTLIT:
            return impl::handle_intlit(prev, beforeprev); 
        case T_FLOATLIT:
            return impl::handle_floatlit(prev, beforeprev); 
        case T_STRINGLIT:
            if (TOKEN_FROM_ID('L', UnknownTokenType) == prev)       // 'L'
                return true;
            break;
#if !defined(WAVE_USE_RE2C_IDL_LEXER)
        case T_LEFTBRACE_ALT:
            return impl::handle_alt_leftbrace(prev, beforeprev); 
        case T_LEFTBRACKET_ALT:
            return impl::handle_alt_leftbracket(prev, beforeprev); 
#else
        case T_FIXEDPOINTLIT:
            return impl::handle_fixedpointlit(prev, beforeprev); 
#endif // !defined(WAVE_USE_RE2C_IDL_LEXER)
        case T_DOT:
            return impl::handle_dot(prev, beforeprev); 
        case T_QUESTION_MARK:
            return impl::handle_questionmark(prev, beforeprev); 
        case T_NEWLINE:
            return impl::handle_newline(prev, beforeprev); 

        case T_LEFTPAREN:
        case T_RIGHTPAREN:
        case T_LEFTBRACKET:
        case T_RIGHTBRACKET:
        case T_SEMICOLON:
        case T_COMMA:
        case T_COLON:
            switch (prev) {
            case T_LEFTPAREN:
            case T_RIGHTPAREN:
            case T_LEFTBRACKET:
            case T_RIGHTBRACKET:
            case T_LEFTBRACE:
            case T_RIGHTBRACE:
                return false;   // no insertion between parens/brackets/braces

            default:
                break;
            }        
            break;
            
        case T_LEFTBRACE:
        case T_RIGHTBRACE:
            switch (prev) {
            case T_LEFTPAREN:
            case T_RIGHTPAREN:
            case T_LEFTBRACKET:
            case T_RIGHTBRACKET:
            case T_LEFTBRACE:
            case T_RIGHTBRACE:
            case T_SEMICOLON:
            case T_COMMA:
            case T_COLON:
                return false;   // no insertion between parens/brackets/braces

            case T_QUESTION_MARK:
                if (T_QUESTION_MARK == beforeprev)
                    return true;
                break;
                
            default:
                break;
            }
            break;
                            
        case T_MINUS:
        case T_MINUSMINUS:
        case T_LESS:
        case T_EQUAL:
        case T_ASSIGN:
        case T_GREATER:
        case T_DIVIDE:
        case T_CHARLIT:
        case T_NOT:
        case T_NOTEQUAL:
#if !defined(WAVE_USE_RE2C_IDL_LEXER)
        case T_DIVIDEASSIGN:
        case T_MINUSASSIGN:
#endif // !defined(WAVE_USE_RE2C_IDL_LEXER)
            if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
                return true;    // ??{op}
            break;

#if !defined(WAVE_USE_RE2C_IDL_LEXER)
        case T_COMPL_ALT:
        case T_OR_ALT:
        case T_AND_ALT:
        case T_NOT_ALT:
        case T_XOR_ALT:
        case T_ANDASSIGN_ALT:
        case T_ORASSIGN_ALT:
        case T_XORASSIGN_ALT:
        case T_NOTEQUAL_ALT:
            if (T_IDENTIFIER == prev || T_NONREPLACABLE_IDENTIFIER == prev ||
                IS_CATEGORY(prev, KeywordTokenType))
                return true;
            break;
#endif // !defined(WAVE_USE_RE2C_IDL_LEXER)
        }

    // else, handle operators separately
        if (IS_CATEGORY(current, OperatorTokenType) && 
            IS_CATEGORY(prev, OperatorTokenType))
        {
            return true;    // operators must be delimited always
        }
        return false;
    }
    void shift_tokens (WAVE_LEXER_NS::token_id next_id)
    {
        beforeprev = prev;
        prev = next_id;
    }
    
private:
    WAVE_LEXER_NS::token_id prev;        // the previous analyzed token
    WAVE_LEXER_NS::token_id beforeprev;  // the token before the previous
};

///////////////////////////////////////////////////////////////////////////////
}   //  namespace util
}   //  namespace wave 

#endif // !defined(INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED)

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

About the Author

Hartmut Kaiser

United States United States
Actively involved in Boost and the development of the Spirit parser construction framework.

| Advertise | Privacy | Mobile
Web02 | 2.8.140721.1 | Last Updated 11 Jan 2004
Article Copyright 2003 by Hartmut Kaiser
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid