aboutsummaryrefslogtreecommitdiff
path: root/lib/Lex/TokenLexer.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Lex/TokenLexer.cpp')
-rw-r--r--lib/Lex/TokenLexer.cpp93
1 files changed, 44 insertions, 49 deletions
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index 0213afcee921..9d03e8d30b8b 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -12,9 +12,9 @@
//===----------------------------------------------------------------------===//
#include "clang/Lex/TokenLexer.h"
-#include "clang/Lex/MacroArgs.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Lex/MacroArgs.h"
#include "clang/Lex/MacroInfo.h"
#include "clang/Lex/Preprocessor.h"
#include "llvm/ADT/SmallString.h"
@@ -37,6 +37,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
ExpandLocEnd = ELEnd;
AtStartOfLine = Tok.isAtStartOfLine();
HasLeadingSpace = Tok.hasLeadingSpace();
+ NextTokGetsSpace = false;
Tokens = &*Macro->tokens_begin();
OwnsTokens = false;
DisableMacroExpansion = false;
@@ -85,8 +86,8 @@ void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
// associated with it.
destroy();
- Macro = 0;
- ActualArgs = 0;
+ Macro = nullptr;
+ ActualArgs = nullptr;
Tokens = TokArray;
OwnsTokens = ownsTokens;
DisableMacroExpansion = disableMacroExpansion;
@@ -95,6 +96,7 @@ void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
ExpandLocStart = ExpandLocEnd = SourceLocation();
AtStartOfLine = false;
HasLeadingSpace = false;
+ NextTokGetsSpace = false;
MacroExpansionStart = SourceLocation();
// Set HasLeadingSpace/AtStartOfLine so that the first token will be
@@ -111,7 +113,7 @@ void TokenLexer::destroy() {
// the expanded tokens.
if (OwnsTokens) {
delete [] Tokens;
- Tokens = 0;
+ Tokens = nullptr;
OwnsTokens = false;
}
@@ -119,13 +121,9 @@ void TokenLexer::destroy() {
if (ActualArgs) ActualArgs->destroy(PP);
}
-/// Remove comma ahead of __VA_ARGS__, if present, according to compiler dialect
-/// settings. Returns true if the comma is removed.
-static bool MaybeRemoveCommaBeforeVaArgs(SmallVectorImpl<Token> &ResultToks,
- bool &NextTokGetsSpace,
- bool HasPasteOperator,
- MacroInfo *Macro, unsigned MacroArgNo,
- Preprocessor &PP) {
+bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
+ SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
+ unsigned MacroArgNo, Preprocessor &PP) {
// Is the macro argument __VA_ARGS__?
if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1)
return false;
@@ -133,7 +131,7 @@ static bool MaybeRemoveCommaBeforeVaArgs(SmallVectorImpl<Token> &ResultToks,
// In Microsoft-compatibility mode, a comma is removed in the expansion
// of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
// not supported by gcc.
- if (!HasPasteOperator && !PP.getLangOpts().MicrosoftMode)
+ if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
return false;
// GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
@@ -179,16 +177,14 @@ void TokenLexer::ExpandFunctionArguments() {
// we install the newly expanded sequence as the new 'Tokens' list.
bool MadeChange = false;
- // NextTokGetsSpace - When this is true, the next token appended to the
- // output list will get a leading space, regardless of whether it had one to
- // begin with or not. This is used for placemarker support.
- bool NextTokGetsSpace = false;
-
for (unsigned i = 0, e = NumTokens; i != e; ++i) {
// If we found the stringify operator, get the argument stringified. The
// preprocessor already verified that the following token is a macro name
// when the #define was parsed.
const Token &CurTok = Tokens[i];
+ if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
+ NextTokGetsSpace = true;
+
if (CurTok.is(tok::hash) || CurTok.is(tok::hashat)) {
int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo());
assert(ArgNo != -1 && "Token following # is not an argument?");
@@ -213,7 +209,7 @@ void TokenLexer::ExpandFunctionArguments() {
// The stringified/charified string leading space flag gets set to match
// the #/#@ operator.
- if (CurTok.hasLeadingSpace() || NextTokGetsSpace)
+ if (NextTokGetsSpace)
Res.setFlag(Token::LeadingSpace);
ResultToks.push_back(Res);
@@ -223,6 +219,13 @@ void TokenLexer::ExpandFunctionArguments() {
continue;
}
+ // Find out if there is a paste (##) operator before or after the token.
+ bool NonEmptyPasteBefore =
+ !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
+ bool PasteBefore = i != 0 && Tokens[i-1].is(tok::hashhash);
+ bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash);
+ assert(!NonEmptyPasteBefore || PasteBefore);
+
// Otherwise, if this is not an argument token, just add the token to the
// output buffer.
IdentifierInfo *II = CurTok.getIdentifierInfo();
@@ -234,7 +237,9 @@ void TokenLexer::ExpandFunctionArguments() {
if (NextTokGetsSpace) {
ResultToks.back().setFlag(Token::LeadingSpace);
NextTokGetsSpace = false;
- }
+ } else if (PasteBefore && !NonEmptyPasteBefore)
+ ResultToks.back().clearFlag(Token::LeadingSpace);
+
continue;
}
@@ -242,18 +247,12 @@ void TokenLexer::ExpandFunctionArguments() {
// input.
MadeChange = true;
- // Otherwise, this is a use of the argument. Find out if there is a paste
- // (##) operator before or after the argument.
- bool NonEmptyPasteBefore =
- !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
- bool PasteBefore = i != 0 && Tokens[i-1].is(tok::hashhash);
- bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash);
- assert(!NonEmptyPasteBefore || PasteBefore);
+ // Otherwise, this is a use of the argument.
// In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
// are no trailing commas if __VA_ARGS__ is empty.
if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
- MaybeRemoveCommaBeforeVaArgs(ResultToks, NextTokGetsSpace,
+ MaybeRemoveCommaBeforeVaArgs(ResultToks,
/*HasPasteOperator=*/false,
Macro, ArgNo, PP))
continue;
@@ -282,7 +281,7 @@ void TokenLexer::ExpandFunctionArguments() {
// behavior by not considering single commas from nested macro
// expansions as argument separators. Set a flag on the token so we can
// test for this later when the macro expansion is processed.
- if (PP.getLangOpts().MicrosoftMode && NumToks == 1 &&
+ if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
ResultToks.back().is(tok::comma))
ResultToks.back().setFlag(Token::IgnoredComma);
@@ -304,13 +303,8 @@ void TokenLexer::ExpandFunctionArguments() {
// before the first token should match the whitespace of the arg
// identifier.
ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
- CurTok.hasLeadingSpace() ||
NextTokGetsSpace);
NextTokGetsSpace = false;
- } else {
- // If this is an empty argument, and if there was whitespace before the
- // formal token, make sure the next token gets whitespace before it.
- NextTokGetsSpace = CurTok.hasLeadingSpace();
}
continue;
}
@@ -358,8 +352,7 @@ void TokenLexer::ExpandFunctionArguments() {
// assembler-with-cpp mode, invalid pastes are allowed through: in this
// case, we do not want the extra whitespace to be added. For example,
// we want ". ## foo" -> ".foo" not ". foo".
- if ((CurTok.hasLeadingSpace() || NextTokGetsSpace) &&
- !NonEmptyPasteBefore)
+ if (NextTokGetsSpace)
ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace);
NextTokGetsSpace = false;
@@ -370,11 +363,9 @@ void TokenLexer::ExpandFunctionArguments() {
// 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
// implement this by eating ## operators when a LHS or RHS expands to
// empty.
- NextTokGetsSpace |= CurTok.hasLeadingSpace();
if (PasteAfter) {
// Discard the argument token and skip (don't copy to the expansion
// buffer) the paste operator after it.
- NextTokGetsSpace |= Tokens[i+1].hasLeadingSpace();
++i;
continue;
}
@@ -385,7 +376,7 @@ void TokenLexer::ExpandFunctionArguments() {
assert(PasteBefore);
if (NonEmptyPasteBefore) {
assert(ResultToks.back().is(tok::hashhash));
- NextTokGetsSpace |= ResultToks.pop_back_val().hasLeadingSpace();
+ ResultToks.pop_back();
}
// If this is the __VA_ARGS__ token, and if the argument wasn't provided,
@@ -393,7 +384,7 @@ void TokenLexer::ExpandFunctionArguments() {
// the ## was a comma, remove the comma. This is a GCC extension which is
// disabled when using -std=c99.
if (ActualArgs->isVarargsElidedUse())
- MaybeRemoveCommaBeforeVaArgs(ResultToks, NextTokGetsSpace,
+ MaybeRemoveCommaBeforeVaArgs(ResultToks,
/*HasPasteOperator=*/true,
Macro, ArgNo, PP);
@@ -425,7 +416,7 @@ bool TokenLexer::Lex(Token &Tok) {
Tok.startToken();
Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
- Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+ Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
if (CurToken == 0)
Tok.setFlag(Token::LeadingEmptyMacro);
return PP.HandleEndOfTokenLexer(Tok);
@@ -479,12 +470,17 @@ bool TokenLexer::Lex(Token &Tok) {
if (isFirstToken) {
Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
- AtStartOfLine = false;
- HasLeadingSpace = false;
+ } else {
+ // If this is not the first token, we may still need to pass through
+ // leading whitespace if we've expanded a macro.
+ if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
+ if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
}
+ AtStartOfLine = false;
+ HasLeadingSpace = false;
// Handle recursive expansion!
- if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != 0) {
+ if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
// Change the kind of this identifier to the appropriate token kind, e.g.
// turning "for" into a keyword.
IdentifierInfo *II = Tok.getIdentifierInfo();
@@ -511,7 +507,7 @@ bool TokenLexer::Lex(Token &Tok) {
/// If this returns true, the caller should immediately return the token.
bool TokenLexer::PasteTokens(Token &Tok) {
SmallString<128> Buffer;
- const char *ResultTokStrPtr = 0;
+ const char *ResultTokStrPtr = nullptr;
SourceLocation StartLoc = Tok.getLocation();
SourceLocation PasteOpLoc;
do {
@@ -620,12 +616,11 @@ bool TokenLexer::PasteTokens(Token &Tok) {
SourceLocation Loc =
SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
// If we're in microsoft extensions mode, downgrade this from a hard
- // error to a warning that defaults to an error. This allows
+ // error to an extension that defaults to an error. This allows
// disabling it.
- PP.Diag(Loc,
- PP.getLangOpts().MicrosoftExt ? diag::err_pp_bad_paste_ms
- : diag::err_pp_bad_paste)
- << Buffer.str();
+ PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
+ : diag::err_pp_bad_paste)
+ << Buffer.str();
}
// An error has occurred so exit loop.
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-09 01:58:17 +0000