Vendor import of stripped clang trunk r366426 (just before thevendor/clang/clang-trunk-r366426

release_90 branch point):

https://llvm.org/svn/llvm-project/cfe/trunk@366426

28 files changed, 504 insertions, 227 deletions

diff --git a/include/clang/Lex/CodeCompletionHandler.h b/include/clang/Lex/CodeCompletionHandler.h
index bef804beed00..bd3e05a36bb3 100644
--- a/include/clang/Lex/CodeCompletionHandler.h
+++ b/include/clang/Lex/CodeCompletionHandler.h
@@ -1,9 +1,8 @@
 //===--- CodeCompletionHandler.h - Preprocessor code completion -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/DependencyDirectivesSourceMinimizer.h b/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
new file mode 100644
index 000000000000..41641078afe4
--- /dev/null
+++ b/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
@@ -0,0 +1,88 @@
+//===- clang/Lex/DependencyDirectivesSourceMinimizer.h -  ----------*- C++ -*-//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This is the interface for minimizing header and source files to the
+/// minimum necessary preprocessor directives for evaluating includes. It
+/// reduces the source down to #define, #include, #import, @import, and any
+/// conditional preprocessor logic that contains one of those.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_DEPENDENCY_DIRECTIVES_SOURCE_MINIMIZER_H
+#define LLVM_CLANG_LEX_DEPENDENCY_DIRECTIVES_SOURCE_MINIMIZER_H
+
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace clang {
+
+class DiagnosticsEngine;
+
+namespace minimize_source_to_dependency_directives {
+
+/// Represents the kind of preprocessor directive or a module declaration that
+/// is tracked by the source minimizer in its token output.
+enum TokenKind {
+  pp_none,
+  pp_include,
+  pp___include_macros,
+  pp_define,
+  pp_undef,
+  pp_import,
+  pp_pragma_import,
+  pp_include_next,
+  pp_if,
+  pp_ifdef,
+  pp_ifndef,
+  pp_elif,
+  pp_else,
+  pp_endif,
+  decl_at_import,
+  pp_eof,
+};
+
+/// Represents a simplified token that's lexed as part of the source
+/// minimization. It's used to track the location of various preprocessor
+/// directives that could potentially have an effect on the depedencies.
+struct Token {
+  /// The kind of token.
+  TokenKind K = pp_none;
+
+  /// Offset into the output byte stream of where the directive begins.
+  int Offset = -1;
+
+  Token(TokenKind K, int Offset) : K(K), Offset(Offset) {}
+};
+
+} // end namespace minimize_source_to_dependency_directives
+
+/// Minimize the input down to the preprocessor directives that might have
+/// an effect on the dependencies for a compilation unit.
+///
+/// This function deletes all non-preprocessor code, and strips anything that
+/// can't affect what gets included. It canonicalizes whitespace where
+/// convenient to stabilize the output against formatting changes in the input.
+///
+/// Clears the output vectors at the beginning of the call.
+///
+/// \returns false on success, true on error. If the diagnostic engine is not
+/// null, an appropriate error is reported using the given input location
+/// with the offset that corresponds to the minimizer's current buffer offset.
+bool minimizeSourceToDependencyDirectives(
+    llvm::StringRef Input, llvm::SmallVectorImpl<char> &Output,
+    llvm::SmallVectorImpl<minimize_source_to_dependency_directives::Token>
+        &Tokens,
+    DiagnosticsEngine *Diags = nullptr,
+    SourceLocation InputSourceLoc = SourceLocation());
+
+} // end namespace clang
+
+#endif // LLVM_CLANG_LEX_DEPENDENCY_DIRECTIVES_SOURCE_MINIMIZER_H
diff --git a/include/clang/Lex/DirectoryLookup.h b/include/clang/Lex/DirectoryLookup.h
index bfb496be5072..7c556ac35175 100644
--- a/include/clang/Lex/DirectoryLookup.h
+++ b/include/clang/Lex/DirectoryLookup.h
@@ -1,9 +1,8 @@
 //===--- DirectoryLookup.h - Info for searching for headers -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -171,6 +170,9 @@ public:
   /// set to true if the file is located in a framework that has been
   /// user-specified to be treated as a system framework.
   ///
+  /// \param [out] IsFrameworkFound For a framework directory set to true if
+  /// specified '.framework' directory is found.
+  ///
   /// \param [out] MappedName if this is a headermap which maps the filename to
   /// a framework include ("Foo.h" -> "Foo/Foo.h"), set the new name to this
   /// vector and point Filename to it.
@@ -181,6 +183,7 @@ public:
                               Module *RequestingModule,
                               ModuleMap::KnownHeader *SuggestedModule,
                               bool &InUserSpecifiedSystemFramework,
+                              bool &IsFrameworkFound,
                               bool &HasBeenMapped,
                               SmallVectorImpl<char> &MappedName) const;
 
@@ -191,7 +194,8 @@ private:
       SmallVectorImpl<char> *RelativePath,
       Module *RequestingModule,
       ModuleMap::KnownHeader *SuggestedModule,
-      bool &InUserSpecifiedSystemFramework) const;
+      bool &InUserSpecifiedSystemFramework,
+      bool &IsFrameworkFound) const;
 
 };
 
diff --git a/include/clang/Lex/ExternalPreprocessorSource.h b/include/clang/Lex/ExternalPreprocessorSource.h
index d849bbd76188..685941b66bd8 100644
--- a/include/clang/Lex/ExternalPreprocessorSource.h
+++ b/include/clang/Lex/ExternalPreprocessorSource.h
@@ -1,9 +1,8 @@
 //===- ExternalPreprocessorSource.h - Abstract Macro Interface --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/HeaderMap.h b/include/clang/Lex/HeaderMap.h
index 793e7edc2752..eca8755d4525 100644
--- a/include/clang/Lex/HeaderMap.h
+++ b/include/clang/Lex/HeaderMap.h
@@ -1,9 +1,8 @@
 //===--- HeaderMap.h - A file that acts like dir of symlinks ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/HeaderMapTypes.h b/include/clang/Lex/HeaderMapTypes.h
index fbaf4baee407..d8881d83d9bf 100644
--- a/include/clang/Lex/HeaderMapTypes.h
+++ b/include/clang/Lex/HeaderMapTypes.h
@@ -1,9 +1,8 @@
 //===- HeaderMapTypes.h - Types for the header map format -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/clang/Lex/HeaderSearch.h b/include/clang/Lex/HeaderSearch.h
index 7c69e219cb57..c5e66242444a 100644
--- a/include/clang/Lex/HeaderSearch.h
+++ b/include/clang/Lex/HeaderSearch.h
@@ -1,9 +1,8 @@
 //===- HeaderSearch.h - Resolve Header File Locations -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -143,22 +142,22 @@ public:
   virtual HeaderFileInfo GetHeaderFileInfo(const FileEntry *FE) = 0;
 };
 
+/// This structure is used to record entries in our framework cache.
+struct FrameworkCacheEntry {
+  /// The directory entry which should be used for the cached framework.
+  const DirectoryEntry *Directory;
+
+  /// Whether this framework has been "user-specified" to be treated as if it
+  /// were a system framework (even if it was found outside a system framework
+  /// directory).
+  bool IsUserSpecifiedSystemFramework;
+};
+
 /// Encapsulates the information needed to find the file referenced
 /// by a \#include or \#include_next, (sub-)framework lookup, etc.
 class HeaderSearch {
   friend class DirectoryLookup;
 
-  /// This structure is used to record entries in our framework cache.
-  struct FrameworkCacheEntry {
-    /// The directory entry which should be used for the cached framework.
-    const DirectoryEntry *Directory;
-
-    /// Whether this framework has been "user-specified" to be treated as if it
-    /// were a system framework (even if it was found outside a system framework
-    /// directory).
-    bool IsUserSpecifiedSystemFramework;
-  };
-
   /// Header-search options used to initialize this header search.
   std::shared_ptr<HeaderSearchOptions> HSOpts;
 
@@ -391,13 +390,19 @@ public:
   ///
   /// \param IsMapped If non-null, and the search involved header maps, set to
   /// true.
+  ///
+  /// \param IsFrameworkFound If non-null, will be set to true if a framework is
+  /// found in any of searched SearchDirs. Will be set to false if a framework
+  /// is found only through header maps. Doesn't guarantee the requested file is
+  /// found.
   const FileEntry *LookupFile(
       StringRef Filename, SourceLocation IncludeLoc, bool isAngled,
       const DirectoryLookup *FromDir, const DirectoryLookup *&CurDir,
       ArrayRef<std::pair<const FileEntry *, const DirectoryEntry *>> Includers,
       SmallVectorImpl<char> *SearchPath, SmallVectorImpl<char> *RelativePath,
       Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule,
-      bool *IsMapped, bool SkipCache = false, bool BuildSystemModule = false);
+      bool *IsMapped, bool *IsFrameworkFound, bool SkipCache = false,
+      bool BuildSystemModule = false);
 
   /// Look up a subframework for the specified \#include file.
   ///
@@ -702,21 +707,31 @@ public:
   /// Retrieve a uniqued framework name.
   StringRef getUniqueFrameworkName(StringRef Framework);
 
-  /// Suggest a path by which the specified file could be found, for
-  /// use in diagnostics to suggest a #include.
+  /// Suggest a path by which the specified file could be found, for use in
+  /// diagnostics to suggest a #include. Returned path will only contain forward
+  /// slashes as separators. MainFile is the absolute path of the file that we
+  /// are generating the diagnostics for. It will try to shorten the path using
+  /// MainFile location, if none of the include search directories were prefix
+  /// of File.
   ///
   /// \param IsSystem If non-null, filled in to indicate whether the suggested
   ///        path is relative to a system header directory.
   std::string suggestPathToFileForDiagnostics(const FileEntry *File,
+                                              llvm::StringRef MainFile,
                                               bool *IsSystem = nullptr);
 
-  /// Suggest a path by which the specified file could be found, for
-  /// use in diagnostics to suggest a #include.
+  /// Suggest a path by which the specified file could be found, for use in
+  /// diagnostics to suggest a #include. Returned path will only contain forward
+  /// slashes as separators. MainFile is the absolute path of the file that we
+  /// are generating the diagnostics for. It will try to shorten the path using
+  /// MainFile location, if none of the include search directories were prefix
+  /// of File.
   ///
   /// \param WorkingDir If non-empty, this will be prepended to search directory
   /// paths that are relative.
   std::string suggestPathToFileForDiagnostics(llvm::StringRef File,
                                               llvm::StringRef WorkingDir,
+                                              llvm::StringRef MainFile,
                                               bool *IsSystem = nullptr);
 
   void PrintStats();
diff --git a/include/clang/Lex/HeaderSearchOptions.h b/include/clang/Lex/HeaderSearchOptions.h
index e5b52b30323f..ed128bce485f 100644
--- a/include/clang/Lex/HeaderSearchOptions.h
+++ b/include/clang/Lex/HeaderSearchOptions.h
@@ -1,9 +1,8 @@
 //===- HeaderSearchOptions.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/clang/Lex/LexDiagnostic.h b/include/clang/Lex/LexDiagnostic.h
index 3a677b834543..86ce162c37ff 100644
--- a/include/clang/Lex/LexDiagnostic.h
+++ b/include/clang/Lex/LexDiagnostic.h
@@ -1,9 +1,8 @@
 //===--- DiagnosticLex.h - Diagnostics for liblex ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/clang/Lex/Lexer.h b/include/clang/Lex/Lexer.h
index a9b10b627320..69cfe62e4bdb 100644
--- a/include/clang/Lex/Lexer.h
+++ b/include/clang/Lex/Lexer.h
@@ -1,9 +1,8 @@
 //===- Lexer.h - C Language Family Lexer ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -383,7 +382,7 @@ public:
     SourceLocation End = getLocForEndOfToken(Range.getEnd(), 0, SM, LangOpts);
     return End.isInvalid() ? CharSourceRange()
                            : CharSourceRange::getCharRange(
-                                 Range.getBegin(), End.getLocWithOffset(-1));
+                                 Range.getBegin(), End);
   }
   static CharSourceRange getAsCharRange(CharSourceRange Range,
                                         const SourceManager &SM,
diff --git a/include/clang/Lex/LiteralSupport.h b/include/clang/Lex/LiteralSupport.h
index 3843a5afd2b0..b9d64c24a00b 100644
--- a/include/clang/Lex/LiteralSupport.h
+++ b/include/clang/Lex/LiteralSupport.h
@@ -1,9 +1,8 @@
 //===--- LiteralSupport.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/MacroArgs.h b/include/clang/Lex/MacroArgs.h
index 853eee2fd779..8806f2d8c656 100644
--- a/include/clang/Lex/MacroArgs.h
+++ b/include/clang/Lex/MacroArgs.h
@@ -1,9 +1,8 @@
 //===--- MacroArgs.h - Formal argument info for Macros ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -113,18 +112,19 @@ public:
   bool isVarargsElidedUse() const { return VarargsElided; }
 
   /// Returns true if the macro was defined with a variadic (ellipsis) parameter
-  /// AND was invoked with at least one token supplied as a variadic argument.
+  /// AND was invoked with at least one token supplied as a variadic argument
+  /// (after pre-expansion).
   ///
   /// \code
   ///   #define F(a)  a
   ///   #define V(a, ...) __VA_OPT__(a)
-  ///   F()    <-- returns false on this invocation.
-  ///   V(,a)  <-- returns true on this invocation.
-  ///   V(,)   <-- returns false on this invocation.
+  ///   F()     <-- returns false on this invocation.
+  ///   V(,a)   <-- returns true on this invocation.
+  ///   V(,)    <-- returns false on this invocation.
+  ///   V(,F()) <-- returns false on this invocation.
   /// \endcode
   ///
-
-  bool invokedWithVariadicArgument(const MacroInfo *const MI) const;
+  bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP);
 
   /// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
   /// tokens into the literal string token that should be produced by the C #
diff --git a/include/clang/Lex/MacroInfo.h b/include/clang/Lex/MacroInfo.h
index a06de132b496..550abf35c841 100644
--- a/include/clang/Lex/MacroInfo.h
+++ b/include/clang/Lex/MacroInfo.h
@@ -1,9 +1,8 @@
 //===- MacroInfo.h - Information about #defined identifiers -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/ModuleLoader.h b/include/clang/Lex/ModuleLoader.h
index 05396dd20597..c93501acb9c2 100644
--- a/include/clang/Lex/ModuleLoader.h
+++ b/include/clang/Lex/ModuleLoader.h
@@ -1,9 +1,8 @@
 //===- ModuleLoader.h - Module Loader Interface -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/ModuleMap.h b/include/clang/Lex/ModuleMap.h
index a38c8d7819db..36e97a16223b 100644
--- a/include/clang/Lex/ModuleMap.h
+++ b/include/clang/Lex/ModuleMap.h
@@ -1,9 +1,8 @@
 //===- ModuleMap.h - Describe the layout of modules -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -521,14 +520,18 @@ public:
                                                bool IsFramework,
                                                bool IsExplicit);
 
-  /// Create a 'global module' for a C++ Modules TS module interface unit.
+  /// Create a global module fragment for a C++ module unit.
   ///
-  /// We model the global module as a submodule of the module interface unit.
-  /// Unfortunately, we can't create the module interface unit's Module until
-  /// later, because we don't know what it will be called.
-  Module *createGlobalModuleForInterfaceUnit(SourceLocation Loc);
+  /// We model the global module fragment as a submodule of the module
+  /// interface unit. Unfortunately, we can't create the module interface
+  /// unit's Module until later, because we don't know what it will be called.
+  Module *createGlobalModuleFragmentForModuleUnit(SourceLocation Loc);
+
+  /// Create a global module fragment for a C++ module interface unit.
+  Module *createPrivateModuleFragmentForInterfaceUnit(Module *Parent,
+                                                      SourceLocation Loc);
 
-  /// Create a new module for a C++ Modules TS module interface unit.
+  /// Create a new module for a C++ module interface unit.
   /// The module must not already exist, and will be configured for the current
   /// compilation.
   ///
diff --git a/include/clang/Lex/MultipleIncludeOpt.h b/include/clang/Lex/MultipleIncludeOpt.h
index ac0dcc7b51c2..7ceb7e53c75d 100644
--- a/include/clang/Lex/MultipleIncludeOpt.h
+++ b/include/clang/Lex/MultipleIncludeOpt.h
@@ -1,9 +1,8 @@
 //===--- MultipleIncludeOpt.h - Header Multiple-Include Optzn ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/clang/Lex/PPCallbacks.h b/include/clang/Lex/PPCallbacks.h
index 2448b34c8af4..f3f3796b1a30 100644
--- a/include/clang/Lex/PPCallbacks.h
+++ b/include/clang/Lex/PPCallbacks.h
@@ -1,9 +1,8 @@
 //===--- PPCallbacks.h - Callbacks for Preprocessor actions -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -133,6 +132,28 @@ public:
                                   SrcMgr::CharacteristicKind FileType) {
   }
 
+  /// Callback invoked whenever a submodule was entered.
+  ///
+  /// \param M The submodule we have entered.
+  ///
+  /// \param ImportLoc The location of import directive token.
+  ///
+  /// \param ForPragma If entering from pragma directive.
+  ///
+  virtual void EnteredSubmodule(Module *M, SourceLocation ImportLoc,
+                                bool ForPragma) { }
+
+  /// Callback invoked whenever a submodule was left.
+  ///
+  /// \param M The submodule we have left.
+  ///
+  /// \param ImportLoc The location of import directive token.
+  ///
+  /// \param ForPragma If entering from pragma directive.
+  ///
+  virtual void LeftSubmodule(Module *M, SourceLocation ImportLoc,
+                             bool ForPragma) { }
+
   /// Callback invoked whenever there was an explicit module-import
   /// syntax.
   ///
@@ -240,6 +261,14 @@ public:
   virtual void PragmaWarningPop(SourceLocation Loc) {
   }
 
+  /// Callback invoked when a \#pragma execution_character_set(push) directive
+  /// is read.
+  virtual void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) {}
+
+  /// Callback invoked when a \#pragma execution_character_set(pop) directive
+  /// is read.
+  virtual void PragmaExecCharsetPop(SourceLocation Loc) {}
+
   /// Callback invoked when a \#pragma clang assume_nonnull begin directive
   /// is read.
   virtual void PragmaAssumeNonNullBegin(SourceLocation Loc) {}
@@ -388,6 +417,18 @@ public:
                                Imported, FileType);
   }
 
+  void EnteredSubmodule(Module *M, SourceLocation ImportLoc,
+                        bool ForPragma) override {
+    First->EnteredSubmodule(M, ImportLoc, ForPragma);
+    Second->EnteredSubmodule(M, ImportLoc, ForPragma);
+  }
+
+  void LeftSubmodule(Module *M, SourceLocation ImportLoc,
+                     bool ForPragma) override {
+    First->LeftSubmodule(M, ImportLoc, ForPragma);
+    Second->LeftSubmodule(M, ImportLoc, ForPragma);
+  }
+
   void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path,
                     const Module *Imported) override {
     First->moduleImport(ImportLoc, Path, Imported);
@@ -478,6 +519,16 @@ public:
     Second->PragmaWarningPop(Loc);
   }
 
+  void PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) override {
+    First->PragmaExecCharsetPush(Loc, Str);
+    Second->PragmaExecCharsetPush(Loc, Str);
+  }
+
+  void PragmaExecCharsetPop(SourceLocation Loc) override {
+    First->PragmaExecCharsetPop(Loc);
+    Second->PragmaExecCharsetPop(Loc);
+  }
+
   void PragmaAssumeNonNullBegin(SourceLocation Loc) override {
     First->PragmaAssumeNonNullBegin(Loc);
     Second->PragmaAssumeNonNullBegin(Loc);
diff --git a/include/clang/Lex/PPConditionalDirectiveRecord.h b/include/clang/Lex/PPConditionalDirectiveRecord.h
index a2ccf1407f79..077437435303 100644
--- a/include/clang/Lex/PPConditionalDirectiveRecord.h
+++ b/include/clang/Lex/PPConditionalDirectiveRecord.h
@@ -1,9 +1,8 @@
 //===--- PPConditionalDirectiveRecord.h - Preprocessing Directives-*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/Pragma.h b/include/clang/Lex/Pragma.h
index fb2942f0916b..e9434269c19c 100644
--- a/include/clang/Lex/Pragma.h
+++ b/include/clang/Lex/Pragma.h
@@ -1,9 +1,8 @@
 //===- Pragma.h - Pragma registration and handling --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,6 +14,7 @@
 #define LLVM_CLANG_LEX_PRAGMA_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include <string>
@@ -47,6 +47,12 @@ class Token;
     PIK___pragma
   };
 
+  /// Describes how and where the pragma was introduced.
+  struct PragmaIntroducer {
+    PragmaIntroducerKind Kind;
+    SourceLocation Loc;
+  };
+
 /// PragmaHandler - Instances of this interface defined to handle the various
 /// pragmas that the language front-end uses.  Each handler optionally has a
 /// name (e.g. "pack") and the HandlePragma method is invoked when a pragma with
@@ -65,7 +71,7 @@ public:
   virtual ~PragmaHandler();
 
   StringRef getName() const { return Name; }
-  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+  virtual void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
                             Token &FirstToken) = 0;
 
   /// getIfNamespace - If this is a namespace, return it.  This is equivalent to
@@ -79,7 +85,7 @@ class EmptyPragmaHandler : public PragmaHandler {
 public:
   explicit EmptyPragmaHandler(StringRef Name = StringRef());
 
-  void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+  void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
                     Token &FirstToken) override;
 };
 
@@ -112,7 +118,7 @@ public:
 
   bool IsEmpty() const { return Handlers.empty(); }
 
-  void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
+  void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer,
                     Token &Tok) override;
 
   PragmaNamespace *getIfNamespace() override { return this; }
diff --git a/include/clang/Lex/PreprocessingRecord.h b/include/clang/Lex/PreprocessingRecord.h
index 027dd3ac5d55..11607811dc8f 100644
--- a/include/clang/Lex/PreprocessingRecord.h
+++ b/include/clang/Lex/PreprocessingRecord.h
@@ -1,9 +1,8 @@
 //===- PreprocessingRecord.h - Record of Preprocessing ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index 64ddb5307fb0..f65b0cda462f 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -1,9 +1,8 @@
 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -34,6 +33,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerUnion.h"
@@ -49,8 +49,8 @@
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
-#include <memory>
 #include <map>
+#include <memory>
 #include <string>
 #include <utility>
 #include <vector>
@@ -72,7 +72,6 @@ class FileEntry;
 class FileManager;
 class HeaderSearch;
 class MacroArgs;
-class MemoryBufferCache;
 class PragmaHandler;
 class PragmaNamespace;
 class PreprocessingRecord;
@@ -126,6 +125,7 @@ class Preprocessor {
   friend class VAOptDefinitionContext;
   friend class VariadicMacroScopeGuard;
 
+  llvm::unique_function<void(const clang::Token &)> OnToken;
   std::shared_ptr<PreprocessorOptions> PPOpts;
   DiagnosticsEngine        *Diags;
   LangOptions       &LangOpts;
@@ -133,7 +133,6 @@ class Preprocessor {
   const TargetInfo *AuxTarget = nullptr;
   FileManager       &FileMgr;
   SourceManager     &SourceMgr;
-  MemoryBufferCache &PCMCache;
   std::unique_ptr<ScratchBuffer> ScratchBuf;
   HeaderSearch      &HeaderInfo;
   ModuleLoader      &TheModuleLoader;
@@ -150,6 +149,7 @@ class Preprocessor {
   IdentifierInfo *Ident__DATE__, *Ident__TIME__;   // __DATE__, __TIME__
   IdentifierInfo *Ident__INCLUDE_LEVEL__;          // __INCLUDE_LEVEL__
   IdentifierInfo *Ident__BASE_FILE__;              // __BASE_FILE__
+  IdentifierInfo *Ident__FILE_NAME__;              // __FILE_NAME__
   IdentifierInfo *Ident__TIMESTAMP__;              // __TIMESTAMP__
   IdentifierInfo *Ident__COUNTER__;                // __COUNTER__
   IdentifierInfo *Ident_Pragma, *Ident__pragma;    // _Pragma, __pragma
@@ -174,6 +174,9 @@ class Preprocessor {
   IdentifierInfo *Ident__is_target_os;             // __is_target_os
   IdentifierInfo *Ident__is_target_environment;    // __is_target_environment
 
+  // Weak, only valid (and set) while InMacroArgs is true.
+  Token* ArgMacro;
+
   SourceLocation DATELoc, TIMELoc;
 
   // Next __COUNTER__ value, starts at 0.
@@ -285,6 +288,84 @@ class Preprocessor {
   /// Whether the last token we lexed was an '@'.
   bool LastTokenWasAt = false;
 
+  /// A position within a C++20 import-seq.
+  class ImportSeq {
+  public:
+    enum State : int {
+      // Positive values represent a number of unclosed brackets.
+      AtTopLevel = 0,
+      AfterTopLevelTokenSeq = -1,
+      AfterExport = -2,
+      AfterImportSeq = -3,
+    };
+
+    ImportSeq(State S) : S(S) {}
+
+    /// Saw any kind of open bracket.
+    void handleOpenBracket() {
+      S = static_cast<State>(std::max<int>(S, 0) + 1);
+    }
+    /// Saw any kind of close bracket other than '}'.
+    void handleCloseBracket() {
+      S = static_cast<State>(std::max<int>(S, 1) - 1);
+    }
+    /// Saw a close brace.
+    void handleCloseBrace() {
+      handleCloseBracket();
+      if (S == AtTopLevel && !AfterHeaderName)
+        S = AfterTopLevelTokenSeq;
+    }
+    /// Saw a semicolon.
+    void handleSemi() {
+      if (atTopLevel()) {
+        S = AfterTopLevelTokenSeq;
+        AfterHeaderName = false;
+      }
+    }
+
+    /// Saw an 'export' identifier.
+    void handleExport() {
+      if (S == AfterTopLevelTokenSeq)
+        S = AfterExport;
+      else if (S <= 0)
+        S = AtTopLevel;
+    }
+    /// Saw an 'import' identifier.
+    void handleImport() {
+      if (S == AfterTopLevelTokenSeq || S == AfterExport)
+        S = AfterImportSeq;
+      else if (S <= 0)
+        S = AtTopLevel;
+    }
+
+    /// Saw a 'header-name' token; do not recognize any more 'import' tokens
+    /// until we reach a top-level semicolon.
+    void handleHeaderName() {
+      if (S == AfterImportSeq)
+        AfterHeaderName = true;
+      handleMisc();
+    }
+
+    /// Saw any other token.
+    void handleMisc() {
+      if (S <= 0)
+        S = AtTopLevel;
+    }
+
+    bool atTopLevel() { return S <= 0; }
+    bool afterImportSeq() { return S == AfterImportSeq; }
+
+  private:
+    State S;
+    /// Whether we're in the pp-import-suffix following the header-name in a
+    /// pp-import. If so, a close-brace is not sufficient to end the
+    /// top-level-token-seq of an import-seq.
+    bool AfterHeaderName = false;
+  };
+
+  /// Our current position within a C++20 import-seq.
+  ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
+
   /// Whether the module import expects an identifier next. Otherwise,
   /// it expects a '.' or ';'.
   bool ModuleImportExpectsIdentifier = false;
@@ -323,6 +404,14 @@ class Preprocessor {
   /// to avoid hitting the same error over and over again.
   bool HasReachedMaxIncludeDepth = false;
 
+  /// The number of currently-active calls to Lex.
+  ///
+  /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
+  /// require asking for multiple additional tokens. This counter makes it
+  /// possible for Lex to detect whether it's producing a token for the end
+  /// of phase 4 of translation or for some other situation.
+  unsigned LexLevel = 0;
+
 public:
   struct PreambleSkipInfo {
     SourceLocation HashTokenLoc;
@@ -777,7 +866,6 @@ private:
 public:
   Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
                DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
-               MemoryBufferCache &PCMCache,
                HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
                IdentifierInfoLookup *IILookup = nullptr,
                bool OwnsHeaderSearch = false,
@@ -817,7 +905,6 @@ public:
   const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
   FileManager &getFileManager() const { return FileMgr; }
   SourceManager &getSourceManager() const { return SourceMgr; }
-  MemoryBufferCache &getPCMCache() const { return PCMCache; }
   HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
 
   IdentifierTable &getIdentifierTable() { return Identifiers; }
@@ -913,6 +1000,13 @@ public:
   }
   /// \}
 
+  /// Register a function that would be called on each token in the final
+  /// expanded token stream.
+  /// This also reports annotation tokens produced by the parser.
+  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+    OnToken = std::move(F);
+  }
+
   bool isMacroDefined(StringRef Id) {
     return isMacroDefined(&Identifiers.get(Id));
   }
@@ -1197,6 +1291,7 @@ public:
   void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
                   MacroArgs *Args);
 
+private:
   /// Add a "macro" context to the top of the include stack,
   /// which will cause the lexer to start returning the specified tokens.
   ///
@@ -1208,18 +1303,24 @@ public:
   /// of tokens has a permanent owner somewhere, so they do not need to be
   /// copied. If it is true, it assumes the array of tokens is allocated with
   /// \c new[] and the Preprocessor will delete[] it.
-private:
+  ///
+  /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
+  /// set, see the flag documentation for details.
   void EnterTokenStream(const Token *Toks, unsigned NumToks,
-                        bool DisableMacroExpansion, bool OwnsTokens);
+                        bool DisableMacroExpansion, bool OwnsTokens,
+                        bool IsReinject);
 
 public:
   void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
-                        bool DisableMacroExpansion) {
-    EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
+                        bool DisableMacroExpansion, bool IsReinject) {
+    EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
+                     IsReinject);
   }
 
-  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
-    EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
+  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
+                        bool IsReinject) {
+    EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
+                     IsReinject);
   }
 
   /// Pop the current lexer/macro exp off the top of the lexer stack.
@@ -1246,24 +1347,6 @@ public:
   /// Disable the last EnableBacktrackAtThisPos call.
   void CommitBacktrackedTokens();
 
-  struct CachedTokensRange {
-    CachedTokensTy::size_type Begin, End;
-  };
-
-private:
-  /// A range of cached tokens that should be erased after lexing
-  /// when backtracking requires the erasure of such cached tokens.
-  Optional<CachedTokensRange> CachedTokenRangeToErase;
-
-public:
-  /// Returns the range of cached tokens that were lexed since
-  /// EnableBacktrackAtThisPos() was previously called.
-  CachedTokensRange LastCachedTokenRange();
-
-  /// Erase the range of cached tokens that were lexed since
-  /// EnableBacktrackAtThisPos() was previously called.
-  void EraseCachedTokens(CachedTokensRange TokenRange);
-
   /// Make Preprocessor re-lex the tokens that were lexed since
   /// EnableBacktrackAtThisPos() was previously called.
   void Backtrack();
@@ -1275,7 +1358,11 @@ public:
   /// Lex the next token for this preprocessor.
   void Lex(Token &Result);
 
-  void LexAfterModuleImport(Token &Result);
+  /// Lex a token, forming a header-name token if possible.
+  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
+
+  bool LexAfterModuleImport(Token &Result);
+  void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
 
   void makeModuleVisible(Module *M, SourceLocation Loc);
 
@@ -1352,6 +1439,7 @@ public:
   /// tokens after phase 5.  As such, it is equivalent to using
   /// 'Lex', not 'LexUnexpandedToken'.
   const Token &LookAhead(unsigned N) {
+    assert(LexLevel == 0 && "cannot use lookahead while lexing");
     if (CachedLexPos + N < CachedTokens.size())
       return CachedTokens[CachedLexPos+N];
     else
@@ -1377,9 +1465,20 @@ public:
   ///
   /// If BackTrack() is called afterwards, the token will remain at the
   /// insertion point.
-  void EnterToken(const Token &Tok) {
-    EnterCachingLexMode();
-    CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
+  /// If \p IsReinject is true, resulting token will have Token::IsReinjected
+  /// flag set. See the flag documentation for details.
+  void EnterToken(const Token &Tok, bool IsReinject) {
+    if (LexLevel) {
+      // It's not correct in general to enter caching lex mode while in the
+      // middle of a nested lexing action.
+      auto TokCopy = llvm::make_unique<Token[]>(1);
+      TokCopy[0] = Tok;
+      EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
+    } else {
+      EnterCachingLexMode();
+      assert(IsReinject && "new tokens in the middle of cached stream");
+      CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
+    }
   }
 
   /// We notify the Preprocessor that if it is caching tokens (because
@@ -1813,11 +1912,15 @@ public:
   /// If not, emit a diagnostic and consume up until the eod.
   /// If \p EnableMacros is true, then we consider macros that expand to zero
   /// tokens as being ok.
-  void CheckEndOfDirective(const char *DirType, bool EnableMacros = false);
+  ///
+  /// \return The location of the end of the directive (the terminating
+  /// newline).
+  SourceLocation CheckEndOfDirective(const char *DirType,
+                                     bool EnableMacros = false);
 
   /// Read and discard all tokens remaining on the current line until
-  /// the tok::eod token is found.
-  void DiscardUntilEndOfDirective();
+  /// the tok::eod token is found. Returns the range of the skipped tokens.
+  SourceRange DiscardUntilEndOfDirective();
 
   /// Returns true if the preprocessor has seen a use of
   /// __DATE__ or __TIME__ in the file so far.
@@ -1855,7 +1958,8 @@ public:
                               SmallVectorImpl<char> *SearchPath,
                               SmallVectorImpl<char> *RelativePath,
                               ModuleMap::KnownHeader *SuggestedModule,
-                              bool *IsMapped, bool SkipCache = false);
+                              bool *IsMapped, bool *IsFrameworkFound,
+                              bool SkipCache = false);
 
   /// Get the DirectoryLookup structure used to find the current
   /// FileEntry, if CurLexer is non-null and if applicable.
@@ -1867,22 +1971,6 @@ public:
   /// Return true if we're in the top-level file, not in a \#include.
   bool isInPrimaryFile() const;
 
-  /// Handle cases where the \#include name is expanded
-  /// from a macro as multiple tokens, which need to be glued together.
-  ///
-  /// This occurs for code like:
-  /// \code
-  ///    \#define FOO <x/y.h>
-  ///    \#include FOO
-  /// \endcode
-  /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
-  ///
-  /// This code concatenates and consumes tokens up to the '>' token.  It
-  /// returns false if the > was found, otherwise it returns true if it finds
-  /// and consumes the EOD marker.
-  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
-                              SourceLocation &End);
-
   /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
   /// followed by EOD.  Return true if the token is not a valid on-off-switch.
   bool LexOnOffSwitch(tok::OnOffSwitch &Result);
@@ -1982,6 +2070,9 @@ private:
 
     /// True if the expression contained identifiers that were undefined.
     bool IncludedUndefinedIds;
+
+    /// The source range for the expression.
+    SourceRange ExprRange;
   };
 
   /// Evaluate an integer constant expression that may occur after a
@@ -2073,6 +2164,7 @@ private:
   }
 
   void EnterCachingLexMode();
+  void EnterCachingLexModeUnchecked();
 
   void ExitCachingLexMode() {
     if (InCachingLexMode())
@@ -2093,12 +2185,32 @@ private:
   void HandleMacroPublicDirective(Token &Tok);
   void HandleMacroPrivateDirective();
 
+  /// An additional notification that can be produced by a header inclusion or
+  /// import to tell the parser what happened.
+  struct ImportAction {
+    enum ActionKind {
+      None,
+      ModuleBegin,
+      ModuleImport,
+      SkippedModuleImport,
+    } Kind;
+    Module *ModuleForHeader = nullptr;
+
+    ImportAction(ActionKind AK, Module *Mod = nullptr)
+        : Kind(AK), ModuleForHeader(Mod) {
+      assert((AK == None || Mod) && "no module for module action");
+    }
+  };
+
   // File inclusion.
-  void HandleIncludeDirective(SourceLocation HashLoc,
-                              Token &Tok,
+  void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
+                              const DirectoryLookup *LookupFrom = nullptr,
+                              const FileEntry *LookupFromFile = nullptr);
+  ImportAction
+  HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
+                              Token &FilenameTok, SourceLocation EndLoc,
                               const DirectoryLookup *LookupFrom = nullptr,
-                              const FileEntry *LookupFromFile = nullptr,
-                              bool isImport = false);
+                              const FileEntry *LookupFromFile = nullptr);
   void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
   void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
   void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
@@ -2179,8 +2291,7 @@ private:
   void HandleElifDirective(Token &ElifToken, const Token &HashToken);
 
   // Pragmas.
-  void HandlePragmaDirective(SourceLocation IntroducerLoc,
-                             PragmaIntroducerKind Introducer);
+  void HandlePragmaDirective(PragmaIntroducer Introducer);
 
 public:
   void HandlePragmaOnce(Token &OnceTok);
diff --git a/include/clang/Lex/PreprocessorLexer.h b/include/clang/Lex/PreprocessorLexer.h
index de918a215302..03b1cc2c10e2 100644
--- a/include/clang/Lex/PreprocessorLexer.h
+++ b/include/clang/Lex/PreprocessorLexer.h
@@ -1,9 +1,8 @@
 //===- PreprocessorLexer.h - C Language Family Lexer ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -49,8 +48,7 @@ protected:
   /// True when parsing \#XXX; turns '\\n' into a tok::eod token.
   bool ParsingPreprocessorDirective = false;
 
-  /// True after \#include; turns \<xx> into a tok::angle_string_literal
-  /// token.
+  /// True after \#include; turns \<xx> or "xxx" into a tok::header_name token.
   bool ParsingFilename = false;
 
   /// True if in raw mode.
@@ -130,11 +128,7 @@ public:
   //===--------------------------------------------------------------------===//
   // Misc. lexing methods.
 
-  /// After the preprocessor has parsed a \#include, lex and
-  /// (potentially) macro expand the filename.
-  ///
-  /// If the sequence parsed is not lexically legal, emit a diagnostic and
-  /// return a result EOD token.
+  /// Lex a token, producing a header-name token if possible.
   void LexIncludeFilename(Token &FilenameTok);
 
   /// Inform the lexer whether or not we are currently lexing a
diff --git a/include/clang/Lex/PreprocessorOptions.h b/include/clang/Lex/PreprocessorOptions.h
index f1ac72c47428..1480548c7fbe 100644
--- a/include/clang/Lex/PreprocessorOptions.h
+++ b/include/clang/Lex/PreprocessorOptions.h
@@ -1,9 +1,8 @@
 //===- PreprocessorOptions.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/clang/Lex/ScratchBuffer.h b/include/clang/Lex/ScratchBuffer.h
index a3d6096821e7..f526f22cb723 100644
--- a/include/clang/Lex/ScratchBuffer.h
+++ b/include/clang/Lex/ScratchBuffer.h
@@ -1,9 +1,8 @@
 //===--- ScratchBuffer.h - Scratch space for forming tokens -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/Token.h b/include/clang/Lex/Token.h
index 85bef728197d..89042a674fec 100644
--- a/include/clang/Lex/Token.h
+++ b/include/clang/Lex/Token.h
@@ -1,9 +1,8 @@
 //===--- Token.h - Token interface ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -71,20 +70,23 @@ class Token {
 public:
   // Various flags set per token:
   enum TokenFlags {
-    StartOfLine   = 0x01,  // At start of line or only after whitespace
-                           // (considering the line after macro expansion).
-    LeadingSpace  = 0x02,  // Whitespace exists before this token (considering
-                           // whitespace after macro expansion).
-    DisableExpand = 0x04,  // This identifier may never be macro expanded.
-    NeedsCleaning = 0x08,  // Contained an escaped newline or trigraph.
+    StartOfLine = 0x01,   // At start of line or only after whitespace
+                          // (considering the line after macro expansion).
+    LeadingSpace = 0x02,  // Whitespace exists before this token (considering
+                          // whitespace after macro expansion).
+    DisableExpand = 0x04, // This identifier may never be macro expanded.
+    NeedsCleaning = 0x08, // Contained an escaped newline or trigraph.
     LeadingEmptyMacro = 0x10, // Empty macro exists before this token.
-    HasUDSuffix = 0x20,    // This string or character literal has a ud-suffix.
-    HasUCN = 0x40,         // This identifier contains a UCN.
-    IgnoredComma = 0x80,   // This comma is not a macro argument separator (MS).
+    HasUDSuffix = 0x20,  // This string or character literal has a ud-suffix.
+    HasUCN = 0x40,       // This identifier contains a UCN.
+    IgnoredComma = 0x80, // This comma is not a macro argument separator (MS).
     StringifiedInMacro = 0x100, // This string or character literal is formed by
                                 // macro stringizing or charizing operator.
     CommaAfterElided = 0x200, // The comma following this token was elided (MS).
     IsEditorPlaceholder = 0x400, // This identifier is a placeholder.
+    IsReinjected = 0x800, // A phase 4 token that was produced before and
+                          // re-added, e.g. via EnterTokenStream. Annotation
+                          // tokens are *not* reinjected.
   };
 
   tok::TokenKind getKind() const { return Kind; }
@@ -329,9 +331,4 @@ struct PPConditionalInfo {
 
 } // end namespace clang
 
-namespace llvm {
-  template <>
-  struct isPodLike<clang::Token> { static const bool value = true; };
-} // end namespace llvm
-
 #endif // LLVM_CLANG_LEX_TOKEN_H
diff --git a/include/clang/Lex/TokenConcatenation.h b/include/clang/Lex/TokenConcatenation.h
index 3199e36f0d26..bd431725d496 100644
--- a/include/clang/Lex/TokenConcatenation.h
+++ b/include/clang/Lex/TokenConcatenation.h
@@ -1,9 +1,8 @@
 //===--- TokenConcatenation.h - Token Concatenation Avoidance ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/clang/Lex/TokenLexer.h b/include/clang/Lex/TokenLexer.h
index 6aae9eec7bfa..4d229ae61067 100644
--- a/include/clang/Lex/TokenLexer.h
+++ b/include/clang/Lex/TokenLexer.h
@@ -1,9 +1,8 @@
 //===- TokenLexer.h - Lex from a token buffer -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -97,6 +96,10 @@ class TokenLexer {
   /// should not be subject to further macro expansion.
   bool DisableMacroExpansion : 1;
 
+  /// When true, the produced tokens have Token::IsReinjected flag set.
+  /// See the flag documentation for details.
+  bool IsReinject : 1;
+
 public:
   /// Create a TokenLexer for the specified macro with the specified actual
   /// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
@@ -112,9 +115,9 @@ public:
   /// specified, this takes ownership of the tokens and delete[]'s them when
   /// the token lexer is empty.
   TokenLexer(const Token *TokArray, unsigned NumToks, bool DisableExpansion,
-             bool ownsTokens, Preprocessor &pp)
+             bool ownsTokens, bool isReinject, Preprocessor &pp)
       : PP(pp), OwnsTokens(false) {
-    Init(TokArray, NumToks, DisableExpansion, ownsTokens);
+    Init(TokArray, NumToks, DisableExpansion, ownsTokens, isReinject);
   }
 
   TokenLexer(const TokenLexer &) = delete;
@@ -133,8 +136,8 @@ public:
   ///
   /// DisableExpansion is true when macro expansion of tokens lexed from this
   /// stream should be disabled.
-  void Init(const Token *TokArray, unsigned NumToks,
-            bool DisableMacroExpansion, bool OwnsTokens);
+  void Init(const Token *TokArray, unsigned NumToks, bool DisableMacroExpansion,
+            bool OwnsTokens, bool IsReinject);
 
   /// If the next token lexed will pop this macro off the
   /// expansion stack, return 2.  If the next unexpanded token is a '(', return
diff --git a/include/clang/Lex/VariadicMacroSupport.h b/include/clang/Lex/VariadicMacroSupport.h
index 3a7a955953f4..989e0ac703c9 100644
--- a/include/clang/Lex/VariadicMacroSupport.h
+++ b/include/clang/Lex/VariadicMacroSupport.h
@@ -1,9 +1,8 @@
 //===- VariadicMacroSupport.h - state machines and scope guards -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -114,6 +113,8 @@ namespace clang {
       UnmatchedOpeningParens.push_back(LParenLoc);
     }
 
+    /// Are we at the top level within the __VA_OPT__?
+    bool isAtTopLevel() const { return UnmatchedOpeningParens.size() == 1; }
   };
 
   /// A class for tracking whether we're inside a VA_OPT during a
@@ -136,7 +137,8 @@ namespace clang {
 
     unsigned StringifyBefore : 1;
     unsigned CharifyBefore : 1;
-
+    unsigned BeginsWithPlaceholder : 1;
+    unsigned EndsWithPlaceholder : 1;
 
     bool hasStringifyBefore() const {
       assert(!isReset() &&
@@ -152,7 +154,8 @@ namespace clang {
   public:
     VAOptExpansionContext(Preprocessor &PP)
         : VAOptDefinitionContext(PP), LeadingSpaceForStringifiedToken(false),
-          StringifyBefore(false), CharifyBefore(false) {
+          StringifyBefore(false), CharifyBefore(false),
+          BeginsWithPlaceholder(false), EndsWithPlaceholder(false) {
       SyntheticEOFToken.startToken();
       SyntheticEOFToken.setKind(tok::eof);
     }
@@ -163,6 +166,8 @@ namespace clang {
       LeadingSpaceForStringifiedToken = false;
       StringifyBefore = false;
       CharifyBefore = false;
+      BeginsWithPlaceholder = false;
+      EndsWithPlaceholder = false;
     }
 
     const Token &getEOFTok() const { return SyntheticEOFToken; }
@@ -175,8 +180,24 @@ namespace clang {
       LeadingSpaceForStringifiedToken = HasLeadingSpace;
     }
 
+    void hasPlaceholderAfterHashhashAtStart() { BeginsWithPlaceholder = true; }
+    void hasPlaceholderBeforeRParen() {
+      if (isAtTopLevel())
+        EndsWithPlaceholder = true;
+    }
 
 
+    bool beginsWithPlaceholder() const {
+      assert(!isReset() &&
+             "Must only be called if the state has not been reset");
+      return BeginsWithPlaceholder;
+    }
+    bool endsWithPlaceholder() const {
+      assert(!isReset() &&
+             "Must only be called if the state has not been reset");
+      return EndsWithPlaceholder;
+    }
+
     bool hasCharifyBefore() const {
       assert(!isReset() &&
              "Must only be called if the state has not been reset");