Another Enum Viewer

// cpre_symtable.cpp: Symbol table management methods

#include "CPreParserImp.h"
#include "cpre_symtable.h"
#include "cpre_stringstream.h"
#include "DLGLexer.h"
#include <algorithm>

///////////////////////////////////////////////////////////////////////
// Macro replacement tokens
// These are not part of the token heirarchy, but are capable
// of holding a token type and some text.  Lists of these are
// used to represent the replacement lists and argument lists
// for macros because they are smaller than regular tokens.
///////////////////////////////////////////////////////////////////////

///////////////////////////////////////////////////////////////////////
// MacroReplacemntToken ctor/dtor
///////////////////////////////////////////////////////////////////////

MacroReplacementToken::MacroReplacementToken(
   ANTLRTokenType tt_, 
   const char* text_
) :
   tokenType((short)tt_)
{
   text = (char*)memPool.New(strlen(text_)+1);
   strcpy(text, text_);
}

MacroReplacementToken::~MacroReplacementToken()
{
   memPool.Delete(text, strlen(text)+1);
}

// copy ctor: Do not copy refcount from rhs!
MacroReplacementToken::MacroReplacementToken(
   const MacroReplacementToken& rhs
) :
   tokenType((short)rhs.getType())
{
   text = (char*)memPool.New(strlen(rhs.getText())+1);
   strcpy(text, rhs.getText());
}

// assignment op: Do not copy refcount from rhs!
MacroReplacementToken& 
MacroReplacementToken::operator=(
   const MacroReplacementToken& rhs
)
{
   if (&rhs != this)
   {
      tokenType = (short)rhs.getType();
      memPool.Delete(text, strlen(text)+1);
      text = (char*)memPool.New(strlen(rhs.getText())+1);
      strcpy(text, rhs.getText());
   }
   return *this;
}

///////////////////////////////////////////////////////////////////////
// Memory allocation pool
///////////////////////////////////////////////////////////////////////
JLMemPool MacroReplacementToken::memPool(MemPoolLimit, MemPoolBlockSize);
JLFreeList MacroReplacementToken::freeList(sizeof(MacroReplacementToken), FreeListBlockSize);

///////////////////////////////////////////////////////////////////////
// MacroReplacementTokenList ctor:
// Create a replacement token list from a regular token list
///////////////////////////////////////////////////////////////////////
MacroReplacementTokenList::MacroReplacementTokenList(
   const TokenList* tokenList
)
{
   for (
      TokenList::const_iterator iter(tokenList->begin());
      iter != tokenList->end();
      iter++
   )
   {
      const ANTLRAbstractToken* tokPtr = (*iter).operator->();
      push_back(new MacroReplacementToken(tokPtr->getType(), tokPtr->getText()));
   }
}

///////////////////////////////////////////////////////////////////////
// CopyToTokenList;
// Fill a regular token list from a replacement token list
///////////////////////////////////////////////////////////////////////
void 
MacroReplacementTokenList::CopyToTokenList(
   TokenList& outputList
) const
{
   outputList.clear();

   for (
      const_iterator iter(begin());
      iter != end();
      iter++
   )
   {
      outputList.push_back(
         FastToken::newToken((*iter)->getType(), (*iter)->getText())
      );
   }
}

///////////////////////////////////////////////////////////////////////
// contentEquals: compare the contents of two MacroReplacementTokenList
///////////////////////////////////////////////////////////////////////
bool 
MacroReplacementTokenList::contentEquals(
   const MacroReplacementTokenList& rhs
) const
{
   CmpReplacementTokenRefs cmpReplacementTokenRefs;
   return
      size() == rhs.size() &&
      equal(begin(), end(), rhs.begin(), cmpReplacementTokenRefs);
}

///////////////////////////////////////////////////////////////////////
//                    Symbol table support                           //
///////////////////////////////////////////////////////////////////////

// Memory allocation for MacroEntrySimple, MacroEntryFunction
JLFreeList MacroEntrySimple::freeList(sizeof(MacroEntrySimple), FreeListBlockSize);
JLFreeList MacroEntryFunction::freeList(sizeof(MacroEntryFunction), FreeListBlockSize);

///////////////////////////////////////////////////////////////////////
// CPreParserImp::EnterSimpleMacro -- Enter a simple macro definition
//    (i.e., a macro that takes no arguments), into the symbol table
//    that is implemented by member simpleSymtable.
//
// Inputs:
//    const ANTLRTokenPtr&   name           The name of the macro to be entered.
// Members used:
//    TokenList              tempTokenList  The list of tokens that is accumulated
//                                          by the token_list* rules.
//
// Return: none
///////////////////////////////////////////////////////////////////////
void CPreParserImp::EnterSimpleMacro(const ANTLRTokenPtr& name)
{
   if (functionSymtable->contains(name))
   {
      HandleError("Redefining function macro to simple macro", name);
      return;
   }

   // Make entry for table
   MacroEntrySimple *entry = new MacroEntrySimple(tempTokenList);
   
   if (simpleSymtable->contains(name))
   {
      // Compare definitions of the two macros
      MacroEntrySimpleRef entryRef;
      entryRef = (*simpleSymtable->find(name)).second;
      assert(entryRef.get() != 0);

      if (*entryRef != *entry)
      {
         HandleError("Invalid redefinition of simple macro", name);
         return;
      }
   }

   // Put entry in table
   simpleSymtable->insert(
      MacroSymbolTableSimple::value_type(name, MacroEntrySimpleRef(entry))
   );

//   printf("*** defining %s = ", name->getText());
//   DumpTokenList(entry->GetReplacementTokenList());
//   printf("\n");
}

///////////////////////////////////////////////////////////////////////
// CPreParserImp::EnterFunctionMacro -- Enter a function macro definition
//    (i.e., a macro that takes arguments), into the symbol table
//    that is implemented by member functionSymtable.
//
// Inputs:
//    const ANTLRTokenPtr   &name          The name of the macro to be entered.
// Members used:
//    TokenList       tempTokenList        The list of tokens that is accumulated
//                                         by the token_list* rules.
//    TokenList       tempIdentifierList   The list of tokens that is accumulated
//                                         by the identifier_list rule.
//
// Return: none
///////////////////////////////////////////////////////////////////////
void CPreParserImp::EnterFunctionMacro(const ANTLRTokenPtr& name)
{
   // Strip the trailing '('
   JLStr functionName(name->getText());
   functionName.erase(functionName.end()-1);

   // Make a new token for the derived name
   ANTLRTokenPtr keyTok(FastToken::newToken(name->getType(), functionName.c_str(), name->getLine()));

//   printf("*** defining %s(", (const char*)function);
//   DumpTempIdentifierList();
//   printf(") ");
//   DumpTempTokenList();
//   printf("\n");

   if (simpleSymtable->contains(keyTok))
   {
      HandleError("Redefining simple macro to function macro", keyTok);
      return;
   }

   // Make entry in symbol table
   MacroEntryFunction* entry = 
      new MacroEntryFunction(tempIdentifierList, tempTokenList);

   // Make sure it isn't a redefinition
   if (functionSymtable->contains(keyTok))
   {
      // Compare definitions of the two macros
      MacroEntryFunctionRef entryRef;
      entryRef = (*functionSymtable->find(keyTok)).second;
      assert(entryRef.get() != 0);

      if (*entryRef != *entry)
      {
         HandleError("Invalid redefinition of function macro", keyTok);
      }
      return;
   }

   functionSymtable->insert(
      MacroSymbolTableFunction::value_type(keyTok, MacroEntryFunctionRef(entry))
   );

//   printf("*** after table insert: args = ");
//   DumpTokenList(entry->GetArgumentTokenList());
//   printf("\n                 replacement = ");
//   DumpTokenList(entry->GetReplacementTokenList());
//   printf("\n");

   #ifdef COLLECT_TOKEN_STATS
      nbrFunctionMacros++;
      nbrFunctionMacroTokens += tempTokenList.entries();
      HistoToken(name);
      for (TokenListIterator iter(tempTokenList); iter(); ) { HistoToken(iter.key()); }
      for (TokenListIterator iter2(tempIdentifierList); iter2(); ) { HistoToken(iter2.key()); }
   #endif
}

///////////////////////////////////////////////////////////////////////
// CPreParserImp::UndefineMacro -- remove a macro from the macro symbol 
//    table(s) implemented by the simpleSymtable and functionSymtable
//    members.
//
// Inputs:
//    const ANTLRTokenPtr&   name     The name of the macro to be removed.
//
// Return: none
///////////////////////////////////////////////////////////////////////
void CPreParserImp::UndefineMacro(const ANTLRTokenPtr& name)
{
   ANTLRTokenKey key(name);
   functionSymtable->erase(key);
   simpleSymtable->erase(key);
}

///////////////////////////////////////////////////////////////////////
// CPreParserImp::MacroIsDefined -- Return whether a macro name is currently
//    defined as a simple or function macro.
//
// Inputs:
//    const ANTLRTokenPtr&   name     The name of the macro to test
//
// Return: true if the macro name is defined.
///////////////////////////////////////////////////////////////////////
bool CPreParserImp::MacroIsDefined(const ANTLRTokenPtr& name)
{
   ANTLRTokenKey key(name);
   return simpleSymtable->contains(key) || functionSymtable->contains(key);
}

///////////////////////////////////////////////////////////////////////
// CPreParserImp::IsFunctionLikeMacro -- Determine whether the token
// is a entered in the function macro symbol table
//
// Inputs:
//    const ANTLRTokenPtr&    name    The token of interest
//
// Return: non-zero if the token is names a function-like macro
///////////////////////////////////////////////////////////////////////
bool 
CPreParserImp::IsFunctionLikeMacro(
   const ANTLRTokenPtr& name
)
{
   ANTLRTokenKey key(name);
   return functionSymtable->contains(key);
}


#ifdef COLLECT_TOKEN_STATS
void 
CPreParserImp::DumpSymtableStats()
{
   fprintf(
      stderr,
      "\nnbrSimpleMacros = %d, nbrFunctionMacros = %d\n"
      "nbrSimpleMacroTokens = %d, nbrFunctionMacroTokens = %d\n",
      nbrSimpleMacros, nbrFunctionMacros, 
      nbrSimpleMacroTokens, nbrFunctionMacroTokens
   );
   fprintf(stderr, "Token size histogram:\n");
   int nbrTokens = 0;
   for (int i = 0; i < tokenHistoSize; i++)
   {
      nbrTokens += tokenSizes[i];
      if (i % 5 == 0 && i != 0) fprintf(stderr, "\n");
      fprintf(stderr, "%2d:%-5d    ", i, tokenSizes[i]);
   }
   fprintf(stderr, "\n");
   fprintf(stderr, "Percent of tokens equal to or smaller than:\n");
   int sumtokens = 0;
   for (i = 0; i < tokenHistoSize; i++)
   {
      sumtokens += tokenSizes[i];
      if (i % 5 == 0 && i != 0) fprintf(stderr, "\n");
      fprintf(stderr, "%2d:%5.1f%%   ", i, 100.0*sumtokens/nbrTokens);
   }
   fprintf(stderr, "\n");
}
#endif


///////////////////////////////////////////////////////////////////////
// CPreParserImp::EnterSymbolDefinition -- define a symbol
//
// Inputs:
//    const JLStr&   definition    A macro definitions, in 
//                                     one of two forms:
//                                         symbol=value, or
//                                         symbol  (define as 1)
// Return:
//     bool      true if the definition succeeded.
//               false if the definition was invalid
///////////////////////////////////////////////////////////////////////
bool
CPreParserImp::EnterSymbolDefinition(
   const JLStr& symbolDef
)
{
   int pos;
   JLStr symbol;
   JLStr value;
   if ((pos = symbolDef.find('=')) != JLStr::npos)
   {
      if (pos > 0)
      {
         symbol = JLStr(symbolDef.begin(), symbolDef.begin()+pos);
         if (pos+1 < (int)symbolDef.length())
         {
            value = JLStr(symbolDef.begin()+pos+1, symbolDef.end());
         } else {
            value = "1";
         }
      } else {
         // Invalid symbol definition -- internal error?
         return false;
      }
   } else {
      symbol = symbolDef;
      value = "1";
   }

   // Token to hand to lexers
   FastToken tok;
   // Make sure that symbol parses as a valid identifier
   CPreStringInput symInput(symbol);
   DLGLexer symScan(&symInput);
   symScan.setToken(&tok);
   // Upcast to stream because @#$%^ lexer hides getToken()
   ANTLRTokenStream* symStream = &symScan;
   ANTLRTokenPtr symToken = symStream->getToken();
   if (!TokenCanBeDefined(symToken))
   {
      // Invalid symbol definition -- internal error?
      return false;
   }

   // Parse the value token
   CPreStringInput valInput(value);
   DLGLexer valScan(&valInput);
   valScan.setToken(&tok);
   ANTLRTokenStream* valStream = &valScan;
   ANTLRTokenPtr valToken = valStream->getToken();
   
   // Enter the macro
   tempTokenList->clear();
   tempTokenList->push_back(valToken);
   EnterSimpleMacro(symToken);

   // success!
   return true;
}

///////////////////////////////////////////////////////////////////////
// CPreParserImp::SetDefinitions -- define a list of simple macros.
//
// Inputs:
//    const JLStr&   definitions   A list of macro definitions,
//                                     separated by semicolons.  Each
//                                     definition is one of two forms:
//                                         symbol=value, or
//                                         symbol  (define as 1)
// Return: none
// Note: This will TOK_NOT clear the preprocessor symbol table.
///////////////////////////////////////////////////////////////////////
void CPreParserImp::SetDefinitions(
   const JLStr& definitions
)
{
   JLStr tmpDefs(definitions);

   int pos;
   // Extract the semicolon-separated symbol definitions
   while ((pos = tmpDefs.find(';')) != JLStr::npos)
   {
      if (pos > 0)
      {
         // Enter string preceeding ';'
         EnterSymbolDefinition(JLStr(tmpDefs.begin(), tmpDefs.begin()+pos));
         // Set string to stuff past ';'
         tmpDefs = JLStr(tmpDefs.begin()+pos+1, tmpDefs.end());
      }
   }

   if (!tmpDefs.empty())
   {
      // Process the last definition
      EnterSymbolDefinition(tmpDefs);
   }
}