/*=============================================================================
Wave: A Standard compliant C++ preprocessor
Re2C based IDL lexer
Copyright (c) 2001-2004 Hartmut Kaiser
http://spirit.sourceforge.net/
Use, modification and distribution is subject to the Boost Software
License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
http://www.boost.org/LICENSE_1_0.txt)
See Copyright.txt for full acknowledgements.
=============================================================================*/
#if !defined(IDL_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED)
#define IDL_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED
#include <string>
#include <cstdio>
#include <cstdarg>
#if defined(BOOST_SPIRIT_DEBUG)
#include <iostream>
#endif // defined(BOOST_SPIRIT_DEBUG)
#include <boost/concept_check.hpp>
#include <boost/spirit/core.hpp>
#include <boost/spirit/core/assert.hpp>
#include "wave/util/file_position.hpp"
#include "wave/cpplexer/validate_universal_char.hpp"
#include "wave/cpplexer/cpplexer_exceptions.hpp"
#include "wave/idllexer/idl_lex_interface.hpp"
#include "wave/idllexer/idl_token_ids.hpp"
#include "wave/idllexer/idl_lex_token.hpp"
#include "wave/idllexer/re2clex/scanner.hpp"
#include "wave/language_support.hpp"
///////////////////////////////////////////////////////////////////////////////
namespace wave {
namespace idllexer {
namespace re2clex {
///////////////////////////////////////////////////////////////////////////////
//
// encapsulation of the re2c based idl lexer
//
///////////////////////////////////////////////////////////////////////////////
template <typename IteratorT, typename PositionT = wave::util::file_position_t>
class lexer
{
public:
typedef char char_t;
typedef Scanner base_t;
typedef wave::idllexer::lex_token<PositionT> token_t;
lexer(IteratorT const &first, IteratorT const &last,
PositionT const &pos, wave::language_support language);
~lexer();
wave::idllexer::lex_token<PositionT> get();
void set_position(PositionT const &pos)
{
filename = pos.get_file();
scanner.line = pos.get_line();
scanner.file_name = filename.c_str();
}
// error reporting from the re2c generated lexer
static int report_error(Scanner *s, char *, ...);
private:
static char const *tok_names[];
Scanner scanner;
WAVE_STRINGTYPE filename;
bool at_eof;
};
///////////////////////////////////////////////////////////////////////////////
// initialize cpp lexer
template <typename IteratorT, typename PositionT>
inline
lexer<IteratorT, PositionT>::lexer(IteratorT const &first,
IteratorT const &last, PositionT const &pos,
wave::language_support language)
: filename(pos.get_file()), at_eof(false)
{
memset(&scanner, '\0', sizeof(Scanner));
scanner.fd = -1;
scanner.eol_offsets = aq_create();
scanner.first = scanner.act = (uchar *)&(*first);
scanner.last = (uchar *)&(*last);
scanner.line = 1; // start with line_no 1
scanner.error_proc = report_error;
scanner.file_name = filename.c_str();
#if defined(WAVE_SUPPORT_MS_EXTENSIONS)
scanner.enable_ms_extensions = 1;
#else
scanner.enable_ms_extensions = 0;
#endif // defined(WAVE_SUPPORT_MS_EXTENSIONS)
#if defined(WAVE_SUPPORT_VARIADICS_PLACEMARKERS)
scanner.act_in_c99_mode = wave::need_c99(language);
#if defined(WAVE_ENABLE_CPP0X_EXTENSIONS)
scanner.act_in_cpp0x_mode = wave::need_cpp0x(language);
#endif
#endif
boost::ignore_unused_variable_warning(language);
}
template <typename IteratorT, typename PositionT>
inline
lexer<IteratorT, PositionT>::~lexer()
{
aq_terminate(scanner.eol_offsets);
free(scanner.bot);
}
///////////////////////////////////////////////////////////////////////////////
// get the next token from the input stream
template <typename IteratorT, typename PositionT>
inline wave::idllexer::lex_token<PositionT>
lexer<IteratorT, PositionT>::get()
{
using namespace wave::idllexer::re2clex;
typedef typename wave::idllexer::lex_token<PositionT>::string_t string_t;
if (at_eof)
return wave::idllexer::lex_token<PositionT>(); // return T_EOI
token_id id = token_id(scan(&scanner));
string_t value((char const *)scanner.tok, scanner.cur-scanner.tok);
if (T_IDENTIFIER == id) {
// test identifier characters for validity (throws if invalid chars found)
wave::cpplexer::impl::validate_identifier_name(value, scanner.line, -1, filename);
}
else if (T_STRINGLIT == id || T_CHARLIT == id) {
// test literal characters for validity (throws if invalid chars found)
wave::cpplexer::impl::validate_literal(value, scanner.line, -1, filename);
}
else if (T_EOF == id) {
// T_EOF is returned as a valid token, the next call will return T_EOI,
// i.e. the actual end of input
at_eof = true;
}
return wave::idllexer::lex_token<PositionT>(id, value,
PositionT(filename, scanner.line, -1));
}
template <typename IteratorT, typename PositionT>
inline int
lexer<IteratorT, PositionT>::report_error(Scanner *s, char *msg, ...)
{
BOOST_SPIRIT_ASSERT(0 != s);
BOOST_SPIRIT_ASSERT(0 != msg);
using namespace std; // some system have vsprintf in namespace std
char buffer[200]; // should be large enough
va_list params;
va_start(params, msg);
vsprintf(buffer, msg, params);
va_end(params);
CPPLEXER_THROW(wave::cpplexer::lexing_exception, generic_lexing_error,
buffer, s->line, -1, s->file_name);
}
///////////////////////////////////////////////////////////////////////////////
//
// lex_functor
//
///////////////////////////////////////////////////////////////////////////////
template <typename IteratorT, typename PositionT = wave::util::file_position_t>
class lex_functor
: public lex_input_interface<typename lexer<IteratorT, PositionT>::token_t>
{
public:
typedef typename lexer<IteratorT, PositionT>::token_t token_t;
lex_functor(IteratorT const &first, IteratorT const &last,
PositionT const &pos, wave::language_support language)
: lexer(first, last, pos, language)
{}
virtual ~lex_functor() {}
// get the next token from the input stream
token_t get() { return lexer.get(); }
void set_position(PositionT const &pos)
{ lexer.set_position(pos); }
private:
lexer<IteratorT, PositionT> lexer;
};
///////////////////////////////////////////////////////////////////////////////
//
// The new_lexer_gen<>::new_lexer function (declared in cpp_slex_token.hpp)
// should be defined inline, if the lex_functor shouldn't be instantiated
// separately from the lex_iterator.
//
// Separate (explicit) instantiation helps to reduce compilation time.
//
///////////////////////////////////////////////////////////////////////////////
#if defined(WAVE_SEPARATE_LEXER_INSTANTIATION)
#define WAVE_RE2C_NEW_LEXER_INLINE
#else
#define WAVE_RE2C_NEW_LEXER_INLINE inline
#endif
} // namespace re2clex
///////////////////////////////////////////////////////////////////////////////
//
// The 'new_lexer' function allows the opaque generation of a new lexer object.
// It is coupled to the iterator type to allow to decouple the lexer/iterator
// configurations at compile time.
//
// This function is declared inside the cpp_slex_token.hpp file, which is
// referenced by the source file calling the lexer and the source file, which
// instantiates the lex_functor. But is is defined here, so it will be
// instantiated only while compiling the source file, which instantiates the
// lex_functor. While the cpp_re2c_token.hpp file may be included everywhere,
// this file (cpp_re2c_lexer.hpp) should be included only once. This allows
// to decouple the lexer interface from the lexer implementation and reduces
// compilation time.
//
///////////////////////////////////////////////////////////////////////////////
template <typename IteratorT, typename PositionT>
WAVE_RE2C_NEW_LEXER_INLINE
lex_input_interface<wave::idllexer::lex_token<PositionT> > *
new_lexer_gen<IteratorT, PositionT>::new_lexer(IteratorT const &first,
IteratorT const &last, PositionT const &pos,
wave::language_support language)
{
return new re2clex::lex_functor<IteratorT, PositionT>(first, last, pos,
language);
}
#undef WAVE_RE2C_NEW_LEXER_INLINE
///////////////////////////////////////////////////////////////////////////////
} // namespace idllexer
} // namespace wave
#endif // !defined(IDL_RE2C_LEXER_HPP_B81A2629_D5B1_4944_A97D_60254182B9A8_INCLUDED)
Submit an article or tip