diff options
Diffstat (limited to 'include/clang/Lex')
28 files changed, 504 insertions, 227 deletions
diff --git a/include/clang/Lex/CodeCompletionHandler.h b/include/clang/Lex/CodeCompletionHandler.h index bef804beed00..bd3e05a36bb3 100644 --- a/include/clang/Lex/CodeCompletionHandler.h +++ b/include/clang/Lex/CodeCompletionHandler.h @@ -1,9 +1,8 @@ //===--- CodeCompletionHandler.h - Preprocessor code completion -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/DependencyDirectivesSourceMinimizer.h b/include/clang/Lex/DependencyDirectivesSourceMinimizer.h new file mode 100644 index 000000000000..41641078afe4 --- /dev/null +++ b/include/clang/Lex/DependencyDirectivesSourceMinimizer.h @@ -0,0 +1,88 @@ +//===- clang/Lex/DependencyDirectivesSourceMinimizer.h - ----------*- C++ -*-// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This is the interface for minimizing header and source files to the +/// minimum necessary preprocessor directives for evaluating includes. It +/// reduces the source down to #define, #include, #import, @import, and any +/// conditional preprocessor logic that contains one of those. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LEX_DEPENDENCY_DIRECTIVES_SOURCE_MINIMIZER_H +#define LLVM_CLANG_LEX_DEPENDENCY_DIRECTIVES_SOURCE_MINIMIZER_H + +#include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +namespace clang { + +class DiagnosticsEngine; + +namespace minimize_source_to_dependency_directives { + +/// Represents the kind of preprocessor directive or a module declaration that +/// is tracked by the source minimizer in its token output. +enum TokenKind { + pp_none, + pp_include, + pp___include_macros, + pp_define, + pp_undef, + pp_import, + pp_pragma_import, + pp_include_next, + pp_if, + pp_ifdef, + pp_ifndef, + pp_elif, + pp_else, + pp_endif, + decl_at_import, + pp_eof, +}; + +/// Represents a simplified token that's lexed as part of the source +/// minimization. It's used to track the location of various preprocessor +/// directives that could potentially have an effect on the depedencies. +struct Token { + /// The kind of token. + TokenKind K = pp_none; + + /// Offset into the output byte stream of where the directive begins. + int Offset = -1; + + Token(TokenKind K, int Offset) : K(K), Offset(Offset) {} +}; + +} // end namespace minimize_source_to_dependency_directives + +/// Minimize the input down to the preprocessor directives that might have +/// an effect on the dependencies for a compilation unit. +/// +/// This function deletes all non-preprocessor code, and strips anything that +/// can't affect what gets included. It canonicalizes whitespace where +/// convenient to stabilize the output against formatting changes in the input. +/// +/// Clears the output vectors at the beginning of the call. +/// +/// \returns false on success, true on error. If the diagnostic engine is not +/// null, an appropriate error is reported using the given input location +/// with the offset that corresponds to the minimizer's current buffer offset. +bool minimizeSourceToDependencyDirectives( + llvm::StringRef Input, llvm::SmallVectorImpl<char> &Output, + llvm::SmallVectorImpl<minimize_source_to_dependency_directives::Token> + &Tokens, + DiagnosticsEngine *Diags = nullptr, + SourceLocation InputSourceLoc = SourceLocation()); + +} // end namespace clang + +#endif // LLVM_CLANG_LEX_DEPENDENCY_DIRECTIVES_SOURCE_MINIMIZER_H diff --git a/include/clang/Lex/DirectoryLookup.h b/include/clang/Lex/DirectoryLookup.h index bfb496be5072..7c556ac35175 100644 --- a/include/clang/Lex/DirectoryLookup.h +++ b/include/clang/Lex/DirectoryLookup.h @@ -1,9 +1,8 @@ //===--- DirectoryLookup.h - Info for searching for headers -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -171,6 +170,9 @@ public: /// set to true if the file is located in a framework that has been /// user-specified to be treated as a system framework. /// + /// \param [out] IsFrameworkFound For a framework directory set to true if + /// specified '.framework' directory is found. + /// /// \param [out] MappedName if this is a headermap which maps the filename to /// a framework include ("Foo.h" -> "Foo/Foo.h"), set the new name to this /// vector and point Filename to it. @@ -181,6 +183,7 @@ public: Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule, bool &InUserSpecifiedSystemFramework, + bool &IsFrameworkFound, bool &HasBeenMapped, SmallVectorImpl<char> &MappedName) const; @@ -191,7 +194,8 @@ private: SmallVectorImpl<char> *RelativePath, Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule, - bool &InUserSpecifiedSystemFramework) const; + bool &InUserSpecifiedSystemFramework, + bool &IsFrameworkFound) const; }; diff --git a/include/clang/Lex/ExternalPreprocessorSource.h b/include/clang/Lex/ExternalPreprocessorSource.h index d849bbd76188..685941b66bd8 100644 --- a/include/clang/Lex/ExternalPreprocessorSource.h +++ b/include/clang/Lex/ExternalPreprocessorSource.h @@ -1,9 +1,8 @@ //===- ExternalPreprocessorSource.h - Abstract Macro Interface --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/HeaderMap.h b/include/clang/Lex/HeaderMap.h index 793e7edc2752..eca8755d4525 100644 --- a/include/clang/Lex/HeaderMap.h +++ b/include/clang/Lex/HeaderMap.h @@ -1,9 +1,8 @@ //===--- HeaderMap.h - A file that acts like dir of symlinks ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/HeaderMapTypes.h b/include/clang/Lex/HeaderMapTypes.h index fbaf4baee407..d8881d83d9bf 100644 --- a/include/clang/Lex/HeaderMapTypes.h +++ b/include/clang/Lex/HeaderMapTypes.h @@ -1,9 +1,8 @@ //===- HeaderMapTypes.h - Types for the header map format -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/clang/Lex/HeaderSearch.h b/include/clang/Lex/HeaderSearch.h index 7c69e219cb57..c5e66242444a 100644 --- a/include/clang/Lex/HeaderSearch.h +++ b/include/clang/Lex/HeaderSearch.h @@ -1,9 +1,8 @@ //===- HeaderSearch.h - Resolve Header File Locations -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -143,22 +142,22 @@ public: virtual HeaderFileInfo GetHeaderFileInfo(const FileEntry *FE) = 0; }; +/// This structure is used to record entries in our framework cache. +struct FrameworkCacheEntry { + /// The directory entry which should be used for the cached framework. + const DirectoryEntry *Directory; + + /// Whether this framework has been "user-specified" to be treated as if it + /// were a system framework (even if it was found outside a system framework + /// directory). + bool IsUserSpecifiedSystemFramework; +}; + /// Encapsulates the information needed to find the file referenced /// by a \#include or \#include_next, (sub-)framework lookup, etc. class HeaderSearch { friend class DirectoryLookup; - /// This structure is used to record entries in our framework cache. - struct FrameworkCacheEntry { - /// The directory entry which should be used for the cached framework. - const DirectoryEntry *Directory; - - /// Whether this framework has been "user-specified" to be treated as if it - /// were a system framework (even if it was found outside a system framework - /// directory). - bool IsUserSpecifiedSystemFramework; - }; - /// Header-search options used to initialize this header search. std::shared_ptr<HeaderSearchOptions> HSOpts; @@ -391,13 +390,19 @@ public: /// /// \param IsMapped If non-null, and the search involved header maps, set to /// true. + /// + /// \param IsFrameworkFound If non-null, will be set to true if a framework is + /// found in any of searched SearchDirs. Will be set to false if a framework + /// is found only through header maps. Doesn't guarantee the requested file is + /// found. const FileEntry *LookupFile( StringRef Filename, SourceLocation IncludeLoc, bool isAngled, const DirectoryLookup *FromDir, const DirectoryLookup *&CurDir, ArrayRef<std::pair<const FileEntry *, const DirectoryEntry *>> Includers, SmallVectorImpl<char> *SearchPath, SmallVectorImpl<char> *RelativePath, Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule, - bool *IsMapped, bool SkipCache = false, bool BuildSystemModule = false); + bool *IsMapped, bool *IsFrameworkFound, bool SkipCache = false, + bool BuildSystemModule = false); /// Look up a subframework for the specified \#include file. /// @@ -702,21 +707,31 @@ public: /// Retrieve a uniqued framework name. StringRef getUniqueFrameworkName(StringRef Framework); - /// Suggest a path by which the specified file could be found, for - /// use in diagnostics to suggest a #include. + /// Suggest a path by which the specified file could be found, for use in + /// diagnostics to suggest a #include. Returned path will only contain forward + /// slashes as separators. MainFile is the absolute path of the file that we + /// are generating the diagnostics for. It will try to shorten the path using + /// MainFile location, if none of the include search directories were prefix + /// of File. /// /// \param IsSystem If non-null, filled in to indicate whether the suggested /// path is relative to a system header directory. std::string suggestPathToFileForDiagnostics(const FileEntry *File, + llvm::StringRef MainFile, bool *IsSystem = nullptr); - /// Suggest a path by which the specified file could be found, for - /// use in diagnostics to suggest a #include. + /// Suggest a path by which the specified file could be found, for use in + /// diagnostics to suggest a #include. Returned path will only contain forward + /// slashes as separators. MainFile is the absolute path of the file that we + /// are generating the diagnostics for. It will try to shorten the path using + /// MainFile location, if none of the include search directories were prefix + /// of File. /// /// \param WorkingDir If non-empty, this will be prepended to search directory /// paths that are relative. std::string suggestPathToFileForDiagnostics(llvm::StringRef File, llvm::StringRef WorkingDir, + llvm::StringRef MainFile, bool *IsSystem = nullptr); void PrintStats(); diff --git a/include/clang/Lex/HeaderSearchOptions.h b/include/clang/Lex/HeaderSearchOptions.h index e5b52b30323f..ed128bce485f 100644 --- a/include/clang/Lex/HeaderSearchOptions.h +++ b/include/clang/Lex/HeaderSearchOptions.h @@ -1,9 +1,8 @@ //===- HeaderSearchOptions.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/clang/Lex/LexDiagnostic.h b/include/clang/Lex/LexDiagnostic.h index 3a677b834543..86ce162c37ff 100644 --- a/include/clang/Lex/LexDiagnostic.h +++ b/include/clang/Lex/LexDiagnostic.h @@ -1,9 +1,8 @@ //===--- DiagnosticLex.h - Diagnostics for liblex ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/clang/Lex/Lexer.h b/include/clang/Lex/Lexer.h index a9b10b627320..69cfe62e4bdb 100644 --- a/include/clang/Lex/Lexer.h +++ b/include/clang/Lex/Lexer.h @@ -1,9 +1,8 @@ //===- Lexer.h - C Language Family Lexer ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -383,7 +382,7 @@ public: SourceLocation End = getLocForEndOfToken(Range.getEnd(), 0, SM, LangOpts); return End.isInvalid() ? CharSourceRange() : CharSourceRange::getCharRange( - Range.getBegin(), End.getLocWithOffset(-1)); + Range.getBegin(), End); } static CharSourceRange getAsCharRange(CharSourceRange Range, const SourceManager &SM, diff --git a/include/clang/Lex/LiteralSupport.h b/include/clang/Lex/LiteralSupport.h index 3843a5afd2b0..b9d64c24a00b 100644 --- a/include/clang/Lex/LiteralSupport.h +++ b/include/clang/Lex/LiteralSupport.h @@ -1,9 +1,8 @@ //===--- LiteralSupport.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/MacroArgs.h b/include/clang/Lex/MacroArgs.h index 853eee2fd779..8806f2d8c656 100644 --- a/include/clang/Lex/MacroArgs.h +++ b/include/clang/Lex/MacroArgs.h @@ -1,9 +1,8 @@ //===--- MacroArgs.h - Formal argument info for Macros ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -113,18 +112,19 @@ public: bool isVarargsElidedUse() const { return VarargsElided; } /// Returns true if the macro was defined with a variadic (ellipsis) parameter - /// AND was invoked with at least one token supplied as a variadic argument. + /// AND was invoked with at least one token supplied as a variadic argument + /// (after pre-expansion). /// /// \code /// #define F(a) a /// #define V(a, ...) __VA_OPT__(a) - /// F() <-- returns false on this invocation. - /// V(,a) <-- returns true on this invocation. - /// V(,) <-- returns false on this invocation. + /// F() <-- returns false on this invocation. + /// V(,a) <-- returns true on this invocation. + /// V(,) <-- returns false on this invocation. + /// V(,F()) <-- returns false on this invocation. /// \endcode /// - - bool invokedWithVariadicArgument(const MacroInfo *const MI) const; + bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP); /// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of /// tokens into the literal string token that should be produced by the C # diff --git a/include/clang/Lex/MacroInfo.h b/include/clang/Lex/MacroInfo.h index a06de132b496..550abf35c841 100644 --- a/include/clang/Lex/MacroInfo.h +++ b/include/clang/Lex/MacroInfo.h @@ -1,9 +1,8 @@ //===- MacroInfo.h - Information about #defined identifiers -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/ModuleLoader.h b/include/clang/Lex/ModuleLoader.h index 05396dd20597..c93501acb9c2 100644 --- a/include/clang/Lex/ModuleLoader.h +++ b/include/clang/Lex/ModuleLoader.h @@ -1,9 +1,8 @@ //===- ModuleLoader.h - Module Loader Interface -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/ModuleMap.h b/include/clang/Lex/ModuleMap.h index a38c8d7819db..36e97a16223b 100644 --- a/include/clang/Lex/ModuleMap.h +++ b/include/clang/Lex/ModuleMap.h @@ -1,9 +1,8 @@ //===- ModuleMap.h - Describe the layout of modules -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -521,14 +520,18 @@ public: bool IsFramework, bool IsExplicit); - /// Create a 'global module' for a C++ Modules TS module interface unit. + /// Create a global module fragment for a C++ module unit. /// - /// We model the global module as a submodule of the module interface unit. - /// Unfortunately, we can't create the module interface unit's Module until - /// later, because we don't know what it will be called. - Module *createGlobalModuleForInterfaceUnit(SourceLocation Loc); + /// We model the global module fragment as a submodule of the module + /// interface unit. Unfortunately, we can't create the module interface + /// unit's Module until later, because we don't know what it will be called. + Module *createGlobalModuleFragmentForModuleUnit(SourceLocation Loc); + + /// Create a global module fragment for a C++ module interface unit. + Module *createPrivateModuleFragmentForInterfaceUnit(Module *Parent, + SourceLocation Loc); - /// Create a new module for a C++ Modules TS module interface unit. + /// Create a new module for a C++ module interface unit. /// The module must not already exist, and will be configured for the current /// compilation. /// diff --git a/include/clang/Lex/MultipleIncludeOpt.h b/include/clang/Lex/MultipleIncludeOpt.h index ac0dcc7b51c2..7ceb7e53c75d 100644 --- a/include/clang/Lex/MultipleIncludeOpt.h +++ b/include/clang/Lex/MultipleIncludeOpt.h @@ -1,9 +1,8 @@ //===--- MultipleIncludeOpt.h - Header Multiple-Include Optzn ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/clang/Lex/PPCallbacks.h b/include/clang/Lex/PPCallbacks.h index 2448b34c8af4..f3f3796b1a30 100644 --- a/include/clang/Lex/PPCallbacks.h +++ b/include/clang/Lex/PPCallbacks.h @@ -1,9 +1,8 @@ //===--- PPCallbacks.h - Callbacks for Preprocessor actions -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -133,6 +132,28 @@ public: SrcMgr::CharacteristicKind FileType) { } + /// Callback invoked whenever a submodule was entered. + /// + /// \param M The submodule we have entered. + /// + /// \param ImportLoc The location of import directive token. + /// + /// \param ForPragma If entering from pragma directive. + /// + virtual void EnteredSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) { } + + /// Callback invoked whenever a submodule was left. + /// + /// \param M The submodule we have left. + /// + /// \param ImportLoc The location of import directive token. + /// + /// \param ForPragma If entering from pragma directive. + /// + virtual void LeftSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) { } + /// Callback invoked whenever there was an explicit module-import /// syntax. /// @@ -240,6 +261,14 @@ public: virtual void PragmaWarningPop(SourceLocation Loc) { } + /// Callback invoked when a \#pragma execution_character_set(push) directive + /// is read. + virtual void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) {} + + /// Callback invoked when a \#pragma execution_character_set(pop) directive + /// is read. + virtual void PragmaExecCharsetPop(SourceLocation Loc) {} + /// Callback invoked when a \#pragma clang assume_nonnull begin directive /// is read. virtual void PragmaAssumeNonNullBegin(SourceLocation Loc) {} @@ -388,6 +417,18 @@ public: Imported, FileType); } + void EnteredSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) override { + First->EnteredSubmodule(M, ImportLoc, ForPragma); + Second->EnteredSubmodule(M, ImportLoc, ForPragma); + } + + void LeftSubmodule(Module *M, SourceLocation ImportLoc, + bool ForPragma) override { + First->LeftSubmodule(M, ImportLoc, ForPragma); + Second->LeftSubmodule(M, ImportLoc, ForPragma); + } + void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path, const Module *Imported) override { First->moduleImport(ImportLoc, Path, Imported); @@ -478,6 +519,16 @@ public: Second->PragmaWarningPop(Loc); } + void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) override { + First->PragmaExecCharsetPush(Loc, Str); + Second->PragmaExecCharsetPush(Loc, Str); + } + + void PragmaExecCharsetPop(SourceLocation Loc) override { + First->PragmaExecCharsetPop(Loc); + Second->PragmaExecCharsetPop(Loc); + } + void PragmaAssumeNonNullBegin(SourceLocation Loc) override { First->PragmaAssumeNonNullBegin(Loc); Second->PragmaAssumeNonNullBegin(Loc); diff --git a/include/clang/Lex/PPConditionalDirectiveRecord.h b/include/clang/Lex/PPConditionalDirectiveRecord.h index a2ccf1407f79..077437435303 100644 --- a/include/clang/Lex/PPConditionalDirectiveRecord.h +++ b/include/clang/Lex/PPConditionalDirectiveRecord.h @@ -1,9 +1,8 @@ //===--- PPConditionalDirectiveRecord.h - Preprocessing Directives-*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/Pragma.h b/include/clang/Lex/Pragma.h index fb2942f0916b..e9434269c19c 100644 --- a/include/clang/Lex/Pragma.h +++ b/include/clang/Lex/Pragma.h @@ -1,9 +1,8 @@ //===- Pragma.h - Pragma registration and handling --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #define LLVM_CLANG_LEX_PRAGMA_H #include "clang/Basic/LLVM.h" +#include "clang/Basic/SourceLocation.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include <string> @@ -47,6 +47,12 @@ class Token; PIK___pragma }; + /// Describes how and where the pragma was introduced. + struct PragmaIntroducer { + PragmaIntroducerKind Kind; + SourceLocation Loc; + }; + /// PragmaHandler - Instances of this interface defined to handle the various /// pragmas that the language front-end uses. Each handler optionally has a /// name (e.g. "pack") and the HandlePragma method is invoked when a pragma with @@ -65,7 +71,7 @@ public: virtual ~PragmaHandler(); StringRef getName() const { return Name; } - virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + virtual void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer, Token &FirstToken) = 0; /// getIfNamespace - If this is a namespace, return it. This is equivalent to @@ -79,7 +85,7 @@ class EmptyPragmaHandler : public PragmaHandler { public: explicit EmptyPragmaHandler(StringRef Name = StringRef()); - void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer, Token &FirstToken) override; }; @@ -112,7 +118,7 @@ public: bool IsEmpty() const { return Handlers.empty(); } - void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer, Token &Tok) override; PragmaNamespace *getIfNamespace() override { return this; } diff --git a/include/clang/Lex/PreprocessingRecord.h b/include/clang/Lex/PreprocessingRecord.h index 027dd3ac5d55..11607811dc8f 100644 --- a/include/clang/Lex/PreprocessingRecord.h +++ b/include/clang/Lex/PreprocessingRecord.h @@ -1,9 +1,8 @@ //===- PreprocessingRecord.h - Record of Preprocessing ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h index 64ddb5307fb0..f65b0cda462f 100644 --- a/include/clang/Lex/Preprocessor.h +++ b/include/clang/Lex/Preprocessor.h @@ -1,9 +1,8 @@ //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,6 +33,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PointerUnion.h" @@ -49,8 +49,8 @@ #include <cassert> #include <cstddef> #include <cstdint> -#include <memory> #include <map> +#include <memory> #include <string> #include <utility> #include <vector> @@ -72,7 +72,6 @@ class FileEntry; class FileManager; class HeaderSearch; class MacroArgs; -class MemoryBufferCache; class PragmaHandler; class PragmaNamespace; class PreprocessingRecord; @@ -126,6 +125,7 @@ class Preprocessor { friend class VAOptDefinitionContext; friend class VariadicMacroScopeGuard; + llvm::unique_function<void(const clang::Token &)> OnToken; std::shared_ptr<PreprocessorOptions> PPOpts; DiagnosticsEngine *Diags; LangOptions &LangOpts; @@ -133,7 +133,6 @@ class Preprocessor { const TargetInfo *AuxTarget = nullptr; FileManager &FileMgr; SourceManager &SourceMgr; - MemoryBufferCache &PCMCache; std::unique_ptr<ScratchBuffer> ScratchBuf; HeaderSearch &HeaderInfo; ModuleLoader &TheModuleLoader; @@ -150,6 +149,7 @@ class Preprocessor { IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__ IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__ IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__ + IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__ IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__ IdentifierInfo *Ident__COUNTER__; // __COUNTER__ IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma @@ -174,6 +174,9 @@ class Preprocessor { IdentifierInfo *Ident__is_target_os; // __is_target_os IdentifierInfo *Ident__is_target_environment; // __is_target_environment + // Weak, only valid (and set) while InMacroArgs is true. + Token* ArgMacro; + SourceLocation DATELoc, TIMELoc; // Next __COUNTER__ value, starts at 0. @@ -285,6 +288,84 @@ class Preprocessor { /// Whether the last token we lexed was an '@'. bool LastTokenWasAt = false; + /// A position within a C++20 import-seq. + class ImportSeq { + public: + enum State : int { + // Positive values represent a number of unclosed brackets. + AtTopLevel = 0, + AfterTopLevelTokenSeq = -1, + AfterExport = -2, + AfterImportSeq = -3, + }; + + ImportSeq(State S) : S(S) {} + + /// Saw any kind of open bracket. + void handleOpenBracket() { + S = static_cast<State>(std::max<int>(S, 0) + 1); + } + /// Saw any kind of close bracket other than '}'. + void handleCloseBracket() { + S = static_cast<State>(std::max<int>(S, 1) - 1); + } + /// Saw a close brace. + void handleCloseBrace() { + handleCloseBracket(); + if (S == AtTopLevel && !AfterHeaderName) + S = AfterTopLevelTokenSeq; + } + /// Saw a semicolon. + void handleSemi() { + if (atTopLevel()) { + S = AfterTopLevelTokenSeq; + AfterHeaderName = false; + } + } + + /// Saw an 'export' identifier. + void handleExport() { + if (S == AfterTopLevelTokenSeq) + S = AfterExport; + else if (S <= 0) + S = AtTopLevel; + } + /// Saw an 'import' identifier. + void handleImport() { + if (S == AfterTopLevelTokenSeq || S == AfterExport) + S = AfterImportSeq; + else if (S <= 0) + S = AtTopLevel; + } + + /// Saw a 'header-name' token; do not recognize any more 'import' tokens + /// until we reach a top-level semicolon. + void handleHeaderName() { + if (S == AfterImportSeq) + AfterHeaderName = true; + handleMisc(); + } + + /// Saw any other token. + void handleMisc() { + if (S <= 0) + S = AtTopLevel; + } + + bool atTopLevel() { return S <= 0; } + bool afterImportSeq() { return S == AfterImportSeq; } + + private: + State S; + /// Whether we're in the pp-import-suffix following the header-name in a + /// pp-import. If so, a close-brace is not sufficient to end the + /// top-level-token-seq of an import-seq. + bool AfterHeaderName = false; + }; + + /// Our current position within a C++20 import-seq. + ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq; + /// Whether the module import expects an identifier next. Otherwise, /// it expects a '.' or ';'. bool ModuleImportExpectsIdentifier = false; @@ -323,6 +404,14 @@ class Preprocessor { /// to avoid hitting the same error over and over again. bool HasReachedMaxIncludeDepth = false; + /// The number of currently-active calls to Lex. + /// + /// Lex is reentrant, and asking for an (end-of-phase-4) token can often + /// require asking for multiple additional tokens. This counter makes it + /// possible for Lex to detect whether it's producing a token for the end + /// of phase 4 of translation or for some other situation. + unsigned LexLevel = 0; + public: struct PreambleSkipInfo { SourceLocation HashTokenLoc; @@ -777,7 +866,6 @@ private: public: Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, - MemoryBufferCache &PCMCache, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup = nullptr, bool OwnsHeaderSearch = false, @@ -817,7 +905,6 @@ public: const TargetInfo *getAuxTargetInfo() const { return AuxTarget; } FileManager &getFileManager() const { return FileMgr; } SourceManager &getSourceManager() const { return SourceMgr; } - MemoryBufferCache &getPCMCache() const { return PCMCache; } HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; } IdentifierTable &getIdentifierTable() { return Identifiers; } @@ -913,6 +1000,13 @@ public: } /// \} + /// Register a function that would be called on each token in the final + /// expanded token stream. + /// This also reports annotation tokens produced by the parser. + void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) { + OnToken = std::move(F); + } + bool isMacroDefined(StringRef Id) { return isMacroDefined(&Identifiers.get(Id)); } @@ -1197,6 +1291,7 @@ public: void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args); +private: /// Add a "macro" context to the top of the include stack, /// which will cause the lexer to start returning the specified tokens. /// @@ -1208,18 +1303,24 @@ public: /// of tokens has a permanent owner somewhere, so they do not need to be /// copied. If it is true, it assumes the array of tokens is allocated with /// \c new[] and the Preprocessor will delete[] it. -private: + /// + /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag + /// set, see the flag documentation for details. void EnterTokenStream(const Token *Toks, unsigned NumToks, - bool DisableMacroExpansion, bool OwnsTokens); + bool DisableMacroExpansion, bool OwnsTokens, + bool IsReinject); public: void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks, - bool DisableMacroExpansion) { - EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true); + bool DisableMacroExpansion, bool IsReinject) { + EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true, + IsReinject); } - void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) { - EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false); + void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion, + bool IsReinject) { + EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false, + IsReinject); } /// Pop the current lexer/macro exp off the top of the lexer stack. @@ -1246,24 +1347,6 @@ public: /// Disable the last EnableBacktrackAtThisPos call. void CommitBacktrackedTokens(); - struct CachedTokensRange { - CachedTokensTy::size_type Begin, End; - }; - -private: - /// A range of cached tokens that should be erased after lexing - /// when backtracking requires the erasure of such cached tokens. - Optional<CachedTokensRange> CachedTokenRangeToErase; - -public: - /// Returns the range of cached tokens that were lexed since - /// EnableBacktrackAtThisPos() was previously called. - CachedTokensRange LastCachedTokenRange(); - - /// Erase the range of cached tokens that were lexed since - /// EnableBacktrackAtThisPos() was previously called. - void EraseCachedTokens(CachedTokensRange TokenRange); - /// Make Preprocessor re-lex the tokens that were lexed since /// EnableBacktrackAtThisPos() was previously called. void Backtrack(); @@ -1275,7 +1358,11 @@ public: /// Lex the next token for this preprocessor. void Lex(Token &Result); - void LexAfterModuleImport(Token &Result); + /// Lex a token, forming a header-name token if possible. + bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true); + + bool LexAfterModuleImport(Token &Result); + void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks); void makeModuleVisible(Module *M, SourceLocation Loc); @@ -1352,6 +1439,7 @@ public: /// tokens after phase 5. As such, it is equivalent to using /// 'Lex', not 'LexUnexpandedToken'. const Token &LookAhead(unsigned N) { + assert(LexLevel == 0 && "cannot use lookahead while lexing"); if (CachedLexPos + N < CachedTokens.size()) return CachedTokens[CachedLexPos+N]; else @@ -1377,9 +1465,20 @@ public: /// /// If BackTrack() is called afterwards, the token will remain at the /// insertion point. - void EnterToken(const Token &Tok) { - EnterCachingLexMode(); - CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok); + /// If \p IsReinject is true, resulting token will have Token::IsReinjected + /// flag set. See the flag documentation for details. + void EnterToken(const Token &Tok, bool IsReinject) { + if (LexLevel) { + // It's not correct in general to enter caching lex mode while in the + // middle of a nested lexing action. + auto TokCopy = llvm::make_unique<Token[]>(1); + TokCopy[0] = Tok; + EnterTokenStream(std::move(TokCopy), 1, true, IsReinject); + } else { + EnterCachingLexMode(); + assert(IsReinject && "new tokens in the middle of cached stream"); + CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok); + } } /// We notify the Preprocessor that if it is caching tokens (because @@ -1813,11 +1912,15 @@ public: /// If not, emit a diagnostic and consume up until the eod. /// If \p EnableMacros is true, then we consider macros that expand to zero /// tokens as being ok. - void CheckEndOfDirective(const char *DirType, bool EnableMacros = false); + /// + /// \return The location of the end of the directive (the terminating + /// newline). + SourceLocation CheckEndOfDirective(const char *DirType, + bool EnableMacros = false); /// Read and discard all tokens remaining on the current line until - /// the tok::eod token is found. - void DiscardUntilEndOfDirective(); + /// the tok::eod token is found. Returns the range of the skipped tokens. + SourceRange DiscardUntilEndOfDirective(); /// Returns true if the preprocessor has seen a use of /// __DATE__ or __TIME__ in the file so far. @@ -1855,7 +1958,8 @@ public: SmallVectorImpl<char> *SearchPath, SmallVectorImpl<char> *RelativePath, ModuleMap::KnownHeader *SuggestedModule, - bool *IsMapped, bool SkipCache = false); + bool *IsMapped, bool *IsFrameworkFound, + bool SkipCache = false); /// Get the DirectoryLookup structure used to find the current /// FileEntry, if CurLexer is non-null and if applicable. @@ -1867,22 +1971,6 @@ public: /// Return true if we're in the top-level file, not in a \#include. bool isInPrimaryFile() const; - /// Handle cases where the \#include name is expanded - /// from a macro as multiple tokens, which need to be glued together. - /// - /// This occurs for code like: - /// \code - /// \#define FOO <x/y.h> - /// \#include FOO - /// \endcode - /// because in this case, "<x/y.h>" is returned as 7 tokens, not one. - /// - /// This code concatenates and consumes tokens up to the '>' token. It - /// returns false if the > was found, otherwise it returns true if it finds - /// and consumes the EOD marker. - bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer, - SourceLocation &End); - /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is /// followed by EOD. Return true if the token is not a valid on-off-switch. bool LexOnOffSwitch(tok::OnOffSwitch &Result); @@ -1982,6 +2070,9 @@ private: /// True if the expression contained identifiers that were undefined. bool IncludedUndefinedIds; + + /// The source range for the expression. + SourceRange ExprRange; }; /// Evaluate an integer constant expression that may occur after a @@ -2073,6 +2164,7 @@ private: } void EnterCachingLexMode(); + void EnterCachingLexModeUnchecked(); void ExitCachingLexMode() { if (InCachingLexMode()) @@ -2093,12 +2185,32 @@ private: void HandleMacroPublicDirective(Token &Tok); void HandleMacroPrivateDirective(); + /// An additional notification that can be produced by a header inclusion or + /// import to tell the parser what happened. + struct ImportAction { + enum ActionKind { + None, + ModuleBegin, + ModuleImport, + SkippedModuleImport, + } Kind; + Module *ModuleForHeader = nullptr; + + ImportAction(ActionKind AK, Module *Mod = nullptr) + : Kind(AK), ModuleForHeader(Mod) { + assert((AK == None || Mod) && "no module for module action"); + } + }; + // File inclusion. - void HandleIncludeDirective(SourceLocation HashLoc, - Token &Tok, + void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok, + const DirectoryLookup *LookupFrom = nullptr, + const FileEntry *LookupFromFile = nullptr); + ImportAction + HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok, + Token &FilenameTok, SourceLocation EndLoc, const DirectoryLookup *LookupFrom = nullptr, - const FileEntry *LookupFromFile = nullptr, - bool isImport = false); + const FileEntry *LookupFromFile = nullptr); void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok); void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok); void HandleImportDirective(SourceLocation HashLoc, Token &Tok); @@ -2179,8 +2291,7 @@ private: void HandleElifDirective(Token &ElifToken, const Token &HashToken); // Pragmas. - void HandlePragmaDirective(SourceLocation IntroducerLoc, - PragmaIntroducerKind Introducer); + void HandlePragmaDirective(PragmaIntroducer Introducer); public: void HandlePragmaOnce(Token &OnceTok); diff --git a/include/clang/Lex/PreprocessorLexer.h b/include/clang/Lex/PreprocessorLexer.h index de918a215302..03b1cc2c10e2 100644 --- a/include/clang/Lex/PreprocessorLexer.h +++ b/include/clang/Lex/PreprocessorLexer.h @@ -1,9 +1,8 @@ //===- PreprocessorLexer.h - C Language Family Lexer ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,8 +48,7 @@ protected: /// True when parsing \#XXX; turns '\\n' into a tok::eod token. bool ParsingPreprocessorDirective = false; - /// True after \#include; turns \<xx> into a tok::angle_string_literal - /// token. + /// True after \#include; turns \<xx> or "xxx" into a tok::header_name token. bool ParsingFilename = false; /// True if in raw mode. @@ -130,11 +128,7 @@ public: //===--------------------------------------------------------------------===// // Misc. lexing methods. - /// After the preprocessor has parsed a \#include, lex and - /// (potentially) macro expand the filename. - /// - /// If the sequence parsed is not lexically legal, emit a diagnostic and - /// return a result EOD token. + /// Lex a token, producing a header-name token if possible. void LexIncludeFilename(Token &FilenameTok); /// Inform the lexer whether or not we are currently lexing a diff --git a/include/clang/Lex/PreprocessorOptions.h b/include/clang/Lex/PreprocessorOptions.h index f1ac72c47428..1480548c7fbe 100644 --- a/include/clang/Lex/PreprocessorOptions.h +++ b/include/clang/Lex/PreprocessorOptions.h @@ -1,9 +1,8 @@ //===- PreprocessorOptions.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/clang/Lex/ScratchBuffer.h b/include/clang/Lex/ScratchBuffer.h index a3d6096821e7..f526f22cb723 100644 --- a/include/clang/Lex/ScratchBuffer.h +++ b/include/clang/Lex/ScratchBuffer.h @@ -1,9 +1,8 @@ //===--- ScratchBuffer.h - Scratch space for forming tokens -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/Token.h b/include/clang/Lex/Token.h index 85bef728197d..89042a674fec 100644 --- a/include/clang/Lex/Token.h +++ b/include/clang/Lex/Token.h @@ -1,9 +1,8 @@ //===--- Token.h - Token interface ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -71,20 +70,23 @@ class Token { public: // Various flags set per token: enum TokenFlags { - StartOfLine = 0x01, // At start of line or only after whitespace - // (considering the line after macro expansion). - LeadingSpace = 0x02, // Whitespace exists before this token (considering - // whitespace after macro expansion). - DisableExpand = 0x04, // This identifier may never be macro expanded. - NeedsCleaning = 0x08, // Contained an escaped newline or trigraph. + StartOfLine = 0x01, // At start of line or only after whitespace + // (considering the line after macro expansion). + LeadingSpace = 0x02, // Whitespace exists before this token (considering + // whitespace after macro expansion). + DisableExpand = 0x04, // This identifier may never be macro expanded. + NeedsCleaning = 0x08, // Contained an escaped newline or trigraph. LeadingEmptyMacro = 0x10, // Empty macro exists before this token. - HasUDSuffix = 0x20, // This string or character literal has a ud-suffix. - HasUCN = 0x40, // This identifier contains a UCN. - IgnoredComma = 0x80, // This comma is not a macro argument separator (MS). + HasUDSuffix = 0x20, // This string or character literal has a ud-suffix. + HasUCN = 0x40, // This identifier contains a UCN. + IgnoredComma = 0x80, // This comma is not a macro argument separator (MS). StringifiedInMacro = 0x100, // This string or character literal is formed by // macro stringizing or charizing operator. CommaAfterElided = 0x200, // The comma following this token was elided (MS). IsEditorPlaceholder = 0x400, // This identifier is a placeholder. + IsReinjected = 0x800, // A phase 4 token that was produced before and + // re-added, e.g. via EnterTokenStream. Annotation + // tokens are *not* reinjected. }; tok::TokenKind getKind() const { return Kind; } @@ -329,9 +331,4 @@ struct PPConditionalInfo { } // end namespace clang -namespace llvm { - template <> - struct isPodLike<clang::Token> { static const bool value = true; }; -} // end namespace llvm - #endif // LLVM_CLANG_LEX_TOKEN_H diff --git a/include/clang/Lex/TokenConcatenation.h b/include/clang/Lex/TokenConcatenation.h index 3199e36f0d26..bd431725d496 100644 --- a/include/clang/Lex/TokenConcatenation.h +++ b/include/clang/Lex/TokenConcatenation.h @@ -1,9 +1,8 @@ //===--- TokenConcatenation.h - Token Concatenation Avoidance ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/clang/Lex/TokenLexer.h b/include/clang/Lex/TokenLexer.h index 6aae9eec7bfa..4d229ae61067 100644 --- a/include/clang/Lex/TokenLexer.h +++ b/include/clang/Lex/TokenLexer.h @@ -1,9 +1,8 @@ //===- TokenLexer.h - Lex from a token buffer -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -97,6 +96,10 @@ class TokenLexer { /// should not be subject to further macro expansion. bool DisableMacroExpansion : 1; + /// When true, the produced tokens have Token::IsReinjected flag set. + /// See the flag documentation for details. + bool IsReinject : 1; + public: /// Create a TokenLexer for the specified macro with the specified actual /// arguments. Note that this ctor takes ownership of the ActualArgs pointer. @@ -112,9 +115,9 @@ public: /// specified, this takes ownership of the tokens and delete[]'s them when /// the token lexer is empty. TokenLexer(const Token *TokArray, unsigned NumToks, bool DisableExpansion, - bool ownsTokens, Preprocessor &pp) + bool ownsTokens, bool isReinject, Preprocessor &pp) : PP(pp), OwnsTokens(false) { - Init(TokArray, NumToks, DisableExpansion, ownsTokens); + Init(TokArray, NumToks, DisableExpansion, ownsTokens, isReinject); } TokenLexer(const TokenLexer &) = delete; @@ -133,8 +136,8 @@ public: /// /// DisableExpansion is true when macro expansion of tokens lexed from this /// stream should be disabled. - void Init(const Token *TokArray, unsigned NumToks, - bool DisableMacroExpansion, bool OwnsTokens); + void Init(const Token *TokArray, unsigned NumToks, bool DisableMacroExpansion, + bool OwnsTokens, bool IsReinject); /// If the next token lexed will pop this macro off the /// expansion stack, return 2. If the next unexpanded token is a '(', return diff --git a/include/clang/Lex/VariadicMacroSupport.h b/include/clang/Lex/VariadicMacroSupport.h index 3a7a955953f4..989e0ac703c9 100644 --- a/include/clang/Lex/VariadicMacroSupport.h +++ b/include/clang/Lex/VariadicMacroSupport.h @@ -1,9 +1,8 @@ //===- VariadicMacroSupport.h - state machines and scope guards -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -114,6 +113,8 @@ namespace clang { UnmatchedOpeningParens.push_back(LParenLoc); } + /// Are we at the top level within the __VA_OPT__? + bool isAtTopLevel() const { return UnmatchedOpeningParens.size() == 1; } }; /// A class for tracking whether we're inside a VA_OPT during a @@ -136,7 +137,8 @@ namespace clang { unsigned StringifyBefore : 1; unsigned CharifyBefore : 1; - + unsigned BeginsWithPlaceholder : 1; + unsigned EndsWithPlaceholder : 1; bool hasStringifyBefore() const { assert(!isReset() && @@ -152,7 +154,8 @@ namespace clang { public: VAOptExpansionContext(Preprocessor &PP) : VAOptDefinitionContext(PP), LeadingSpaceForStringifiedToken(false), - StringifyBefore(false), CharifyBefore(false) { + StringifyBefore(false), CharifyBefore(false), + BeginsWithPlaceholder(false), EndsWithPlaceholder(false) { SyntheticEOFToken.startToken(); SyntheticEOFToken.setKind(tok::eof); } @@ -163,6 +166,8 @@ namespace clang { LeadingSpaceForStringifiedToken = false; StringifyBefore = false; CharifyBefore = false; + BeginsWithPlaceholder = false; + EndsWithPlaceholder = false; } const Token &getEOFTok() const { return SyntheticEOFToken; } @@ -175,8 +180,24 @@ namespace clang { LeadingSpaceForStringifiedToken = HasLeadingSpace; } + void hasPlaceholderAfterHashhashAtStart() { BeginsWithPlaceholder = true; } + void hasPlaceholderBeforeRParen() { + if (isAtTopLevel()) + EndsWithPlaceholder = true; + } + bool beginsWithPlaceholder() const { + assert(!isReset() && + "Must only be called if the state has not been reset"); + return BeginsWithPlaceholder; + } + bool endsWithPlaceholder() const { + assert(!isReset() && + "Must only be called if the state has not been reset"); + return EndsWithPlaceholder; + } + bool hasCharifyBefore() const { assert(!isReset() && "Must only be called if the state has not been reset"); |